From 1162b506bcb4bea4754afcc1a5273fb8c6033f9f Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Mon, 25 May 2026 17:05:12 +0700 Subject: [PATCH 01/16] feat: expose ingest_subtree_sst + replace_commitment_tree_subtree_root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two new public methods to enable snapshot-based bootstrap of CommitmentTree subtrees without paying the per-write WAL + fsync cost of the runtime seeder path: 1. `RocksDbStorage::ingest_subtree_sst` (also re-exposed on `GroveDb`) — bulk-ingest a single SST file (produced by `SstFileWriter`) into a named column family via `IngestExternalFile`. Pinned `allow_global_seqno=false` and `snapshot_consistency=false` for safe defaults. 2. `GroveDb::replace_commitment_tree_subtree_root` — extracts the parent-Merk tail of `commitment_tree_insert` as a public method, accepting a caller- provided `new_combined_root` instead of computing it from an append. Used in conjunction with (1) to apply a precomputed shielded-pool snapshot at devnet genesis. Bootstrap pattern (for the caller, e.g. drive-abci's shielded_snapshot module): - Use (1) to ingest the subtree's underlying RocksDB state from a SST file. - Open StorageContext at the subtree path, reload CommitmentTree, compute combined_root via `compute_commitment_tree_state_root`, verify it matches the snapshot header's recorded value. - Use (2) to write the parent Merk leaf with the verified combined_root. Neither method performs cross-validation on its own; misuse will produce an inconsistent Merk tree. Intended for snapshot-based bootstrap only; normal append flow must continue to go through `commitment_tree_insert`. Co-Authored-By: Claude Opus 4.7 --- grovedb/src/lib.rs | 25 ++++++ grovedb/src/operations/commitment_tree.rs | 97 ++++++++++++++++++++++- storage/src/rocksdb_storage/storage.rs | 47 ++++++++++- 3 files changed, 166 insertions(+), 3 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 6a77abf7b..09636f05e 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -343,6 +343,31 @@ impl GroveDb { Ok(()) } + /// Bulk-ingest a single SST file (produced by `rocksdb::SstFileWriter`) + /// into the named column family of the underlying RocksDB. + /// + /// Delegates to + /// [`grovedb_storage::rocksdb_storage::RocksDbStorage::ingest_subtree_sst`]. + /// Intended for snapshot-based bootstrap of a single subtree's storage + /// state — see that method's docs for safety contract. + /// + /// CF name for ordinary data storage is the default CF + /// (`rocksdb::DEFAULT_COLUMN_FAMILY_NAME`). Aux/roots/meta CFs are also + /// valid targets if a snapshot tool happens to cover them. + /// + /// This call bypasses any open transaction. The caller is responsible for + /// transaction semantics at a higher layer (e.g. only call when the + /// destination subtree is known empty, and rely on InitChain + /// abort = wipe-and-restart for failure recovery). + pub fn ingest_subtree_sst( + &self, + cf_name: &str, + sst_path: &std::path::Path, + ) -> Result<(), Error> { + self.db.ingest_subtree_sst(cf_name, sst_path)?; + Ok(()) + } + /// Opens the transactional Merk at the given path. Returns CostResult. fn open_transactional_merk_at_path<'db, 'b, B>( &'db self, diff --git a/grovedb/src/operations/commitment_tree.rs b/grovedb/src/operations/commitment_tree.rs index 8fc5d7021..17d521bfa 100644 --- a/grovedb/src/operations/commitment_tree.rs +++ b/grovedb/src/operations/commitment_tree.rs @@ -30,7 +30,7 @@ use grovedb_version::version::GroveVersion; use crate::{ batch::{GroveOp, QualifiedGroveDbOp}, util::TxRef, - Element, Error, GroveDb, Transaction, TransactionArg, + Element, ElementFlags, Error, GroveDb, Transaction, TransactionArg, }; // ── Helpers ────────────────────────────────────────────────────────────── @@ -238,6 +238,101 @@ impl GroveDb { .wrap_with_cost(cost) } + /// Replace the child hash of a CommitmentTree subtree leaf in its parent + /// Merk with a caller-provided `new_combined_root`, simultaneously + /// updating the leaf's `total_count` and `flags`. + /// + /// This is the parent-Merk tail of [`Self::commitment_tree_insert`] + /// extracted as a public method, intended for **snapshot-based bootstrap** + /// only (e.g. the shielded-pool genesis snapshot at devnet `InitChain` + /// time). Normal append flow MUST go through `commitment_tree_insert`. + /// + /// The caller is responsible for ensuring `new_combined_root` actually + /// matches the subtree's underlying state. The intended caller pattern: + /// 1. Ingest the subtree's storage state into the underlying RocksDB + /// (typically via [`grovedb_storage::rocksdb_storage::RocksDbStorage::ingest_subtree_sst`]). + /// 2. Open a `StorageContext` at the subtree path and reconstruct + /// `CommitmentTree` from it. + /// 3. Compute `combined_root` via + /// `grovedb_commitment_tree::compute_commitment_tree_state_root`, + /// binding the Sinsemilla anchor to the bulk-state root. + /// 4. Call this method with the verified `combined_root`. + /// + /// Mismatches between `new_combined_root` and what re-reading the subtree + /// would compute will produce an inconsistent Merk tree (parent's + /// recorded child hash diverges from actual subtree state). This method + /// does NOT detect such mismatches. + /// + /// `chunk_power` must match the value the BulkAppendTree state was built + /// with — same caveat: not validated here. + pub fn replace_commitment_tree_subtree_root<'b, B, P>( + &self, + path: P, + key: &[u8], + new_total_count: u64, + chunk_power: u8, + flags: Option, + new_combined_root: [u8; 32], + transaction: TransactionArg, + grove_version: &GroveVersion, + ) -> CostResult<(), Error> + where + B: AsRef<[u8]> + 'b, + P: Into>, + { + let path: SubtreePath = path.into(); + let mut cost = OperationCost::default(); + let tx = TxRef::new(&self.db, transaction); + + let batch = StorageBatch::new(); + let mut parent_merk = cost_return_on_error!( + &mut cost, + self.open_transactional_merk_at_path( + path.clone(), + tx.as_ref(), + Some(&batch), + grove_version, + ) + ); + + let updated_element = + Element::new_commitment_tree(new_total_count, chunk_power, flags); + + cost_return_on_error_into!( + &mut cost, + updated_element.insert_subtree( + &mut parent_merk, + key, + new_combined_root, + None, + grove_version, + ) + ); + + let mut merk_cache = HashMap::new(); + merk_cache.insert(path.clone(), parent_merk); + + cost_return_on_error!( + &mut cost, + self.propagate_changes_with_transaction( + merk_cache, + path, + tx.as_ref(), + &batch, + grove_version, + ) + ); + + cost_return_on_error!( + &mut cost, + self.db + .commit_multi_context_batch(batch, Some(tx.as_ref())) + .map_err(Into::into) + ); + + tx.commit_local().wrap_with_cost(cost) + } + /// Get the Orchard `Anchor` for a CommitmentTree subtree. /// /// Returns the anchor directly as an `orchard::tree::Anchor`, suitable for diff --git a/storage/src/rocksdb_storage/storage.rs b/storage/src/rocksdb_storage/storage.rs index 9ff20c4c4..9a1493e47 100644 --- a/storage/src/rocksdb_storage/storage.rs +++ b/storage/src/rocksdb_storage/storage.rs @@ -40,8 +40,8 @@ use grovedb_path::SubtreePath; use integer_encoding::VarInt; use lazy_static::lazy_static; use rocksdb::{ - checkpoint::Checkpoint, ColumnFamily, ColumnFamilyDescriptor, OptimisticTransactionDB, - Transaction, WriteBatchWithTransaction, DEFAULT_COLUMN_FAMILY_NAME, + checkpoint::Checkpoint, ColumnFamily, ColumnFamilyDescriptor, IngestExternalFileOptions, + OptimisticTransactionDB, Transaction, WriteBatchWithTransaction, DEFAULT_COLUMN_FAMILY_NAME, }; use super::{PrefixedRocksDbImmediateStorageContext, PrefixedRocksDbTransactionContext}; @@ -494,6 +494,49 @@ impl RocksDbStorage { } } + /// Bulk-ingest a single SST file (produced by `rocksdb::SstFileWriter`) + /// into the named column family. + /// + /// Used by snapshot-based bootstrap (e.g. the shielded-pool genesis + /// snapshot) to load a precomputed subtree's keys without paying the + /// per-write WAL + fsync cost. The SST must be sorted and its key range + /// must NOT overlap with any keys already in the CF — otherwise ingest + /// fails. For genesis-time usage this is satisfied by definition (the + /// target subtree is empty when this is called). + /// + /// Security notes (set by this method, not caller-configurable): + /// - `allow_global_seqno=false`: rejects ingests that would inject a + /// global sequence number, preventing a malicious snapshot from + /// reordering its writes relative to subsequent transactional writes. + /// - `snapshot_consistency=false`: snapshot-based bootstrap runs before + /// any reader could hold a RocksDB snapshot of the empty state. + /// + /// The ingest happens at the DB level and bypasses any open transaction. + /// Callers must arrange for txn semantics at a higher layer. + pub fn ingest_subtree_sst( + &self, + cf_name: &str, + sst_path: &Path, + ) -> Result<(), Error> { + let cf_handle = self + .db + .cf_handle(cf_name) + .ok_or(Error::StorageError(format!( + "ingest_subtree_sst: missing CF {cf_name}" + )))?; + let mut opts = IngestExternalFileOptions::default(); + opts.set_allow_global_seqno(false); + opts.set_snapshot_consistency(false); + self.db + .ingest_external_file_cf_opts(&cf_handle, &opts, vec![sst_path]) + .map_err(|e| { + Error::StorageError(format!( + "ingest_subtree_sst({cf_name}, {}) failed: {e}", + sst_path.display() + )) + }) + } + /// Clears all data from the database using range deletion on each /// column family. Uses a single range tombstone per CF instead of /// iterating and deleting every key individually. From 04f2d4243872b65fbec33650e15d85571df385e1 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Mon, 25 May 2026 17:11:22 +0700 Subject: [PATCH 02/16] feat: expose raw_storage() on GroveDb Escape hatch for snapshot/replication tooling that needs to use the public StorageContext API directly (raw_iter etc.) without paying the typed element-layer overhead. Co-Authored-By: Claude Opus 4.7 --- grovedb/src/lib.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 09636f05e..727256fe9 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -343,6 +343,22 @@ impl GroveDb { Ok(()) } + /// Reborrow the underlying [`grovedb_storage::rocksdb_storage::RocksDbStorage`] + /// for callers that need to use the public [`grovedb_storage::Storage`] + /// trait directly — notably to open a [`grovedb_storage::StorageContext`] + /// at a path for raw iteration or low-level reads. + /// + /// This is intended for snapshot/replication tooling that needs to walk + /// a subtree's raw RocksDB state without going through GroveDb's typed + /// element API. Normal callers should NOT use this — go through GroveDb's + /// typed operations (`insert`, `get`, `commitment_tree_*`) instead. + /// + /// Stability: this is an escape hatch. The exact `RocksDbStorage` shape + /// is subject to change as grovedb's internals evolve. + pub fn raw_storage(&self) -> &grovedb_storage::rocksdb_storage::RocksDbStorage { + &self.db + } + /// Bulk-ingest a single SST file (produced by `rocksdb::SstFileWriter`) /// into the named column family of the underlying RocksDB. /// From 48a000bbf8102b2f5f40c6c990d14feed3fd4f25 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 25 May 2026 14:19:01 +0700 Subject: [PATCH 03/16] feat(commitment-tree): frontier-less bulk seeding for sync/scale testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `test-seeding`-gated `append_raw_without_frontier` and `append_many_without_frontier` on `CommitmentTree`. These populate the underlying BulkAppendTree at blake3 speed WITHOUT updating the Sinsemilla frontier, so a devnet shielded pool can be pre-loaded with a large N of filler notes without paying the per-note Pallas/Sinsemilla hashing (or the full Drive insert path). A frontier-less seeded tree has no valid Orchard anchor (so seeded notes aren't spendable), but BulkAppendTree chunk proofs — authenticated by the blake3 bulk state root, not the frontier — still verify, which is exactly what client wallet sync exercises. Under `test-seeding`, `CommitmentTree::open` tolerates the empty-frontier/populated-bulk shape so seeded state round-trips; production behavior is unchanged when the feature is off. The grovedb crate forwards this via `commitment_tree_test_seeding`. For devnet/benchmark seeding only — never enable in production. Co-Authored-By: Claude Opus 4.7 (1M context) --- grovedb-commitment-tree/Cargo.toml | 10 + .../src/commitment_tree/mod.rs | 211 +++++++++++++++++- .../src/commitment_tree/tests.rs | 142 ++++++++++++ grovedb-commitment-tree/src/lib.rs | 2 + grovedb/Cargo.toml | 5 + 5 files changed, 365 insertions(+), 5 deletions(-) diff --git a/grovedb-commitment-tree/Cargo.toml b/grovedb-commitment-tree/Cargo.toml index 17f30b65a..6469393b5 100644 --- a/grovedb-commitment-tree/Cargo.toml +++ b/grovedb-commitment-tree/Cargo.toml @@ -15,6 +15,16 @@ default = [] server = ["grovedb-storage", "grovedb-bulk-append-tree/storage"] client = ["shardtree"] sqlite = ["client", "rusqlite"] +# TEST / DEVNET ONLY — do not enable in production. +# +# Adds frontier-less seeding methods (`append_raw_without_frontier`, +# `append_many_without_frontier`) that populate the underlying BulkAppendTree +# WITHOUT updating the Sinsemilla frontier. This lets benchmarks pre-populate a +# shielded pool with a large N of filler notes at blake3 speed, skipping the +# per-note Pallas/Sinsemilla hashing. The resulting tree has NO valid Orchard +# anchor, so seeded notes are not spendable; chunk proofs (authenticated by the +# blake3 bulk state root) still verify, which is what client sync exercises. +test-seeding = ["server"] [dependencies] orchard = { git = "https://github.com/dashpay/orchard.git", rev = "898258d76aab2822249492aede59a02d49278fff", features = ["circuit"] } diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs index 28b461b66..9ceac86a3 100644 --- a/grovedb-commitment-tree/src/commitment_tree/mod.rs +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -198,13 +198,23 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { // Validate that the frontier and bulk tree agree on the number of // appended items. A mismatch indicates a partial commit or data // corruption. + // + // Exception: with the `test-seeding` feature, the frontier-less seeding + // methods (`append_*_without_frontier`) populate the BulkAppendTree + // while deliberately leaving the frontier empty. Tolerate exactly that + // shape — an empty frontier alongside a populated bulk tree — so seeded + // devnet state can be re-opened. Any other mismatch is still rejected. let frontier_size = frontier.tree_size(); if frontier_size != total_count { - return Err(CommitmentTreeError::InvalidData(format!( - "frontier tree_size ({}) != bulk tree total_count ({})", - frontier_size, total_count - ))) - .wrap_with_cost(cost); + let frontier_less_seed = + cfg!(feature = "test-seeding") && frontier_size == 0 && total_count > 0; + if !frontier_less_seed { + return Err(CommitmentTreeError::InvalidData(format!( + "frontier tree_size ({}) != bulk tree total_count ({})", + frontier_size, total_count + ))) + .wrap_with_cost(cost); + } } Ok(Self { @@ -426,3 +436,194 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { self.bulk_tree.chunk_count() } } + +/// Result of a single frontier-less append via +/// [`CommitmentTree::append_raw_without_frontier`]. +/// +/// Mirrors [`CommitmentAppendResult`] but omits `sinsemilla_root`, because the +/// Sinsemilla frontier is intentionally left untouched. Only available with the +/// `test-seeding` feature. +#[cfg(feature = "test-seeding")] +#[derive(Debug, Clone)] +pub struct FrontierLessAppendResult { + /// The BulkAppendTree state root after the append (the Merk child hash). + pub bulk_state_root: [u8; 32], + /// The 0-based global position of the appended value. + pub global_position: u64, + /// Number of blake3 hash calls performed during the bulk append. + pub hash_count: u32, + /// Whether compaction (epoch flush) occurred during this append. + pub compacted: bool, +} + +/// Summary of a frontier-less bulk seed via +/// [`CommitmentTree::append_many_without_frontier`]. +/// +/// Only available with the `test-seeding` feature. +#[cfg(feature = "test-seeding")] +#[derive(Debug, Clone)] +pub struct BulkSeedSummary { + /// Number of notes appended during this call. + pub appended: u64, + /// The tree's total item count after the call. + pub total_count: u64, + /// The final BulkAppendTree state root (the Merk child hash). This binds + /// only the bulk data; a frontier-less seeded tree has no valid Orchard + /// anchor. + pub bulk_state_root: [u8; 32], + /// Total blake3 hash calls performed across all appends and compactions. + pub hash_count: u64, + /// Number of epoch compactions (chunk finalizations) triggered. + pub compactions: u64, +} + +/// Frontier-less seeding API — **test / devnet only**. +/// +/// These methods append note entries to the underlying [`BulkAppendTree`] +/// WITHOUT updating the Sinsemilla [`CommitmentFrontier`]. They exist to +/// pre-populate the shielded pool with a large number of filler notes for +/// sync/scale benchmarking without paying the per-note Pallas/Sinsemilla +/// hashing cost (or the full Drive insert path). +/// +/// # Consequences +/// +/// - The tree has **no valid Orchard anchor** afterwards: [`anchor`] and +/// [`root_hash`] reflect an empty frontier, so notes seeded this way are +/// **not spendable**. The BulkAppendTree chunk proofs — authenticated by the +/// blake3 `bulk_state_root` rather than the frontier — are unaffected and +/// still verify, which is exactly what client sync exercises. +/// - `cmx` values are **not** validated as Pallas field elements (unlike +/// [`append`] and [`append_raw`]), so arbitrary 32-byte filler is accepted. +/// - A tree seeded this way can only be re-[`open`]ed by a build that also +/// enables `test-seeding`, which relaxes the frontier/bulk consistency check +/// for the empty-frontier case. +/// +/// [`anchor`]: CommitmentTree::anchor +/// [`root_hash`]: CommitmentTree::root_hash +/// [`append`]: CommitmentTree::append +/// [`append_raw`]: CommitmentTree::append_raw +/// [`open`]: CommitmentTree::open +#[cfg(feature = "test-seeding")] +impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { + /// Append a single note (`cmx || rho || payload`) to the BulkAppendTree + /// **without** updating the Sinsemilla frontier. + /// + /// The payload length must equal [`ciphertext_payload_size::()`]. The + /// `cmx` is stored verbatim and is *not* validated as a Pallas field + /// element. See the impl-level docs for the consequences. + pub fn append_raw_without_frontier( + &mut self, + cmx: [u8; 32], + rho: [u8; 32], + payload: &[u8], + ) -> CostResult { + let mut cost = OperationCost::default(); + + // Validate payload size — kept because it keeps stored entries + // well-formed for chunk proofs and client deserialization. (cmx is + // intentionally not validated as a Pallas field element here.) + let expected = ciphertext_payload_size::(); + if payload.len() != expected { + return Err(CommitmentTreeError::InvalidPayloadSize { + expected, + actual: payload.len(), + }) + .wrap_with_cost(cost); + } + + let mut item_value = Vec::with_capacity(64 + payload.len()); + item_value.extend_from_slice(&cmx); + item_value.extend_from_slice(&rho); + item_value.extend_from_slice(payload); + + let bulk_result = match self.bulk_tree.append(&item_value) { + Ok(r) => r, + Err(e) => { + return Err(CommitmentTreeError::InvalidData(format!( + "bulk append: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + cost.hash_node_calls += bulk_result.hash_count; + + Ok(FrontierLessAppendResult { + bulk_state_root: bulk_result.state_root, + global_position: bulk_result.global_position, + hash_count: bulk_result.hash_count, + compacted: bulk_result.compacted, + }) + .wrap_with_cost(cost) + } + + /// Bulk-append many notes to the BulkAppendTree **without** updating the + /// Sinsemilla frontier. + /// + /// Each item is `(cmx, rho, payload)`. The iterator is consumed lazily, so + /// a generator can stream a large `N` without materializing it. MMR nodes + /// buffered during compaction are flushed via [`commit_mmr`] before + /// returning, so the caller does not need a separate `commit_mmr()` call. + /// The frontier is left untouched; callers typically set the parent + /// `Element::CommitmentTree(total_count, ..)` from the returned summary. + /// + /// [`commit_mmr`]: CommitmentTree::commit_mmr + pub fn append_many_without_frontier( + &mut self, + notes: I, + ) -> CostResult + where + I: IntoIterator)>, + { + let mut cost = OperationCost::default(); + let mut appended: u64 = 0; + let mut hash_count: u64 = 0; + let mut compactions: u64 = 0; + let mut bulk_state_root = [0u8; 32]; + + for (cmx, rho, payload) in notes { + let r = match self + .append_raw_without_frontier(cmx, rho, &payload) + .unwrap_add_cost(&mut cost) + { + Ok(r) => r, + Err(e) => return Err(e).wrap_with_cost(cost), + }; + appended += 1; + hash_count += u64::from(r.hash_count); + if r.compacted { + compactions += 1; + } + bulk_state_root = r.bulk_state_root; + } + + // Flush MMR nodes staged during compaction so the seeded state is + // fully persisted; callers don't need a separate commit_mmr(). + if let Err(e) = self.commit_mmr() { + return Err(e).wrap_with_cost(cost); + } + + if appended == 0 { + // Nothing appended — report the current (unchanged) state root. + bulk_state_root = match self.bulk_tree.compute_current_state_root() { + Ok(r) => r, + Err(e) => { + return Err(CommitmentTreeError::InvalidData(format!( + "state root: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + } + + Ok(BulkSeedSummary { + appended, + total_count: self.bulk_tree.total_count, + bulk_state_root, + hash_count, + compactions, + }) + .wrap_with_cost(cost) + } +} diff --git a/grovedb-commitment-tree/src/commitment_tree/tests.rs b/grovedb-commitment-tree/src/commitment_tree/tests.rs index 1b6442bbc..861884164 100644 --- a/grovedb-commitment-tree/src/commitment_tree/tests.rs +++ b/grovedb-commitment-tree/src/commitment_tree/tests.rs @@ -1021,4 +1021,146 @@ mod storage_tests { "tree should not have been mutated by invalid cmx" ); } + + // ── Frontier-less seeding (test-seeding feature) ────────────────────── + + /// A correctly-sized DashMemo payload filled with a deterministic pattern. + #[cfg(feature = "test-seeding")] + fn seed_payload(index: u8) -> Vec { + let mut p = vec![0u8; ciphertext_payload_size::()]; + p[0] = index; + p[1] = 0x5D; + p + } + + #[cfg(feature = "test-seeding")] + #[test] + fn test_append_raw_without_frontier_does_not_touch_frontier() { + let ctx = MockDataStorageContext::new(); + let mut ct = + CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); + + let r = ct + .append_raw_without_frontier(test_leaf(0), test_rho(0), &seed_payload(0)) + .value + .expect("frontier-less append should succeed"); + + assert_eq!(r.global_position, 0, "first append is position 0"); + assert_eq!(ct.total_count(), 1, "bulk tree advanced"); + // Frontier untouched: still empty. + assert_eq!(ct.tree_size(), 0, "frontier must remain empty"); + assert_eq!(ct.position(), None, "frontier has no position"); + assert_eq!( + ct.root_hash(), + CommitmentFrontier::new().root_hash(), + "anchor must equal the empty-frontier root" + ); + } + + #[cfg(feature = "test-seeding")] + #[test] + fn test_append_raw_without_frontier_rejects_wrong_payload_size() { + let ctx = MockDataStorageContext::new(); + let mut ct = + CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); + + let result = ct.append_raw_without_frontier(test_leaf(0), test_rho(0), &[0u8; 7]); + assert!( + result.value.is_err(), + "should reject a payload of the wrong size" + ); + assert_eq!(ct.total_count(), 0, "tree must not be mutated on rejection"); + } + + #[cfg(feature = "test-seeding")] + #[test] + fn test_append_raw_without_frontier_accepts_non_pallas_cmx() { + let ctx = MockDataStorageContext::new(); + let mut ct = + CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); + + // All-0xFF is NOT a valid Pallas field element; the frontier-less path + // accepts it anyway (no frontier validation). + let result = ct.append_raw_without_frontier([0xFF; 32], test_rho(1), &seed_payload(1)); + assert!( + result.value.is_ok(), + "frontier-less append must accept arbitrary cmx filler" + ); + assert_eq!(ct.total_count(), 1); + } + + #[cfg(feature = "test-seeding")] + #[test] + fn test_append_many_without_frontier_seeds_and_reopens() { + let ctx = MockDataStorageContext::new(); + let mut ct = + CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); + + const N: u64 = 12; + let notes = (0..N).map(|i| (test_leaf(i), test_rho(i as u8), seed_payload(i as u8))); + let summary = ct + .append_many_without_frontier(notes) + .value + .expect("bulk seed should succeed"); + + assert_eq!(summary.appended, N); + assert_eq!(summary.total_count, N); + assert_eq!(ct.total_count(), N); + // TEST_CHUNK_POWER=1 → epoch_size 2, so seeding 12 notes finalizes + // several chunks. + assert!( + summary.compactions > 0, + "small epoch size should trigger compactions" + ); + assert!(summary.hash_count > 0, "appends perform blake3 hashes"); + // Frontier left empty by design. + assert_eq!(ct.tree_size(), 0, "frontier must remain empty"); + + // The summary's bulk_state_root matches a fresh computation. + let live_root = ct + .bulk_tree + .compute_current_state_root() + .expect("state root"); + assert_eq!(summary.bulk_state_root, live_root); + + // Re-open the seeded (frontier-less) tree: tolerated under test-seeding. + let storage = ct.bulk_tree.dense_tree.storage; + let loaded = CommitmentTree::<_, DashMemo>::open(N, TEST_CHUNK_POWER, storage) + .value + .expect("open should tolerate empty frontier under test-seeding"); + assert_eq!(loaded.total_count(), N, "reopened total_count matches"); + assert_eq!(loaded.tree_size(), 0, "reopened frontier still empty"); + assert_eq!( + loaded + .bulk_tree + .compute_current_state_root() + .expect("state root"), + live_root, + "bulk state root survives the round-trip" + ); + } + + #[cfg(feature = "test-seeding")] + #[test] + fn test_append_many_without_frontier_empty_input() { + let ctx = MockDataStorageContext::new(); + let mut ct = + CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); + + let summary = ct + .append_many_without_frontier(std::iter::empty()) + .value + .expect("empty seed should succeed"); + + assert_eq!(summary.appended, 0); + assert_eq!(summary.total_count, 0); + assert_eq!(summary.compactions, 0); + assert_eq!( + summary.bulk_state_root, + ct.bulk_tree + .compute_current_state_root() + .expect("state root"), + "empty seed reports the current state root" + ); + } } diff --git a/grovedb-commitment-tree/src/lib.rs b/grovedb-commitment-tree/src/lib.rs index 7257a4ff9..74e4cdc75 100644 --- a/grovedb-commitment-tree/src/lib.rs +++ b/grovedb-commitment-tree/src/lib.rs @@ -67,6 +67,8 @@ pub use commitment_tree::{ ciphertext_payload_size, deserialize_ciphertext, serialize_ciphertext, CommitmentAppendResult, CommitmentTree, COMMITMENT_TREE_DATA_KEY, }; +#[cfg(feature = "test-seeding")] +pub use commitment_tree::{BulkSeedSummary, FrontierLessAppendResult}; pub use error::CommitmentTreeError; #[cfg(feature = "server")] pub use grovedb_bulk_append_tree::{ diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index 3e3d069b2..1abe4be2d 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -108,6 +108,11 @@ verify = [ ] estimated_costs = ["full"] zk_client = ["grovedb-commitment-tree", "grovedb-commitment-tree/client"] +# TEST / DEVNET ONLY — forwards `grovedb-commitment-tree/test-seeding`. Exposes +# the frontier-less commitment-tree seeding methods and relaxes the +# frontier/bulk consistency check in `CommitmentTree::open` so a node can read +# seeded (anchor-less) devnet state. Never enable in production. +commitment_tree_test_seeding = ["minimal", "grovedb-commitment-tree/test-seeding"] grovedbg = [ "grovedbg-types", "tokio", From 3fd39906043c3ca97a8f2a37ca25033a2fa16ab8 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 25 May 2026 14:25:38 +0700 Subject: [PATCH 04/16] fix(commitment-tree): reject frontier-less seeding on a non-empty frontier Guard both `append_raw_without_frontier` and `append_many_without_frontier`: advancing the bulk tree while the frontier already has leaves would leave `frontier_size < total_count`, a mismatch `open` cannot tolerate (it only accepts an empty frontier), producing unreopenable state. Reject it instead. Addresses CodeRabbit review on #751. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/commitment_tree/mod.rs | 22 +++++++++++ .../src/commitment_tree/tests.rs | 38 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs index 9ceac86a3..9760f35ef 100644 --- a/grovedb-commitment-tree/src/commitment_tree/mod.rs +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -519,6 +519,18 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { ) -> CostResult { let mut cost = OperationCost::default(); + // Frontier-less seeding only makes sense on an empty frontier. If the + // frontier already has leaves, advancing the bulk tree here would leave + // `frontier_size < total_count`, a mismatch `open` cannot tolerate + // (it only accepts an empty frontier) — i.e. unreopenable state. Reject + // it instead of silently corrupting the tree. + if self.frontier.tree_size() != 0 { + return Err(CommitmentTreeError::InvalidData( + "frontier-less seeding requires an empty frontier".to_string(), + )) + .wrap_with_cost(cost); + } + // Validate payload size — kept because it keeps stored entries // well-formed for chunk proofs and client deserialization. (cmx is // intentionally not validated as a Pallas field element here.) @@ -576,6 +588,16 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { I: IntoIterator)>, { let mut cost = OperationCost::default(); + + // Fail fast before consuming the iterator: frontier-less seeding is only + // valid on an empty frontier (see `append_raw_without_frontier`). + if self.frontier.tree_size() != 0 { + return Err(CommitmentTreeError::InvalidData( + "frontier-less seeding requires an empty frontier".to_string(), + )) + .wrap_with_cost(cost); + } + let mut appended: u64 = 0; let mut hash_count: u64 = 0; let mut compactions: u64 = 0; diff --git a/grovedb-commitment-tree/src/commitment_tree/tests.rs b/grovedb-commitment-tree/src/commitment_tree/tests.rs index 861884164..4fae209d6 100644 --- a/grovedb-commitment-tree/src/commitment_tree/tests.rs +++ b/grovedb-commitment-tree/src/commitment_tree/tests.rs @@ -1140,6 +1140,44 @@ mod storage_tests { ); } + #[cfg(feature = "test-seeding")] + #[test] + fn test_frontier_less_append_rejected_on_non_empty_frontier() { + let ctx = MockDataStorageContext::new(); + let mut ct = + CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); + + // Build a non-empty frontier via a normal append first. + ct.append(test_leaf(0), test_rho(0), &test_ciphertext(0)) + .value + .expect("normal append should succeed"); + assert_eq!(ct.tree_size(), 1, "frontier should be non-empty"); + let total_before = ct.total_count(); + + // Both frontier-less entry points must refuse to advance the bulk tree. + let single = ct.append_raw_without_frontier(test_leaf(1), test_rho(1), &seed_payload(1)); + assert!( + single.value.is_err(), + "single frontier-less append must be rejected on a non-empty frontier" + ); + + let bulk = ct.append_many_without_frontier(std::iter::once(( + test_leaf(2), + test_rho(2), + seed_payload(2), + ))); + assert!( + bulk.value.is_err(), + "bulk frontier-less seed must be rejected on a non-empty frontier" + ); + + assert_eq!( + ct.total_count(), + total_before, + "rejected frontier-less appends must not mutate the tree" + ); + } + #[cfg(feature = "test-seeding")] #[test] fn test_append_many_without_frontier_empty_input() { From b7b2090601b002e4a6462ef4d2505c9ff42e47f4 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 25 May 2026 14:32:18 +0700 Subject: [PATCH 05/16] refactor(commitment-tree): rename feature to test-seeding-ct; drop open() check under it Rename the seeding feature `test-seeding` -> `test-seeding-ct` (and the grovedb forwarding feature to match). Under the feature, drop the frontier/bulk consistency check in `CommitmentTree::open` entirely rather than tolerating only the empty-frontier case. This lets a frontier-less-seeded tree have real, frontier-tracked notes added on top via the normal `append` path and still reopen at any `(frontier_size, total_count)` pair. Production behavior is unchanged when the feature is off. Co-Authored-By: Claude Opus 4.7 (1M context) --- grovedb-commitment-tree/Cargo.toml | 2 +- .../src/commitment_tree/mod.rs | 47 +++++++-------- .../src/commitment_tree/tests.rs | 59 +++++++++++++++---- grovedb-commitment-tree/src/lib.rs | 2 +- grovedb/Cargo.toml | 6 +- 5 files changed, 78 insertions(+), 38 deletions(-) diff --git a/grovedb-commitment-tree/Cargo.toml b/grovedb-commitment-tree/Cargo.toml index 6469393b5..ec9c734ff 100644 --- a/grovedb-commitment-tree/Cargo.toml +++ b/grovedb-commitment-tree/Cargo.toml @@ -24,7 +24,7 @@ sqlite = ["client", "rusqlite"] # per-note Pallas/Sinsemilla hashing. The resulting tree has NO valid Orchard # anchor, so seeded notes are not spendable; chunk proofs (authenticated by the # blake3 bulk state root) still verify, which is what client sync exercises. -test-seeding = ["server"] +test-seeding-ct = ["server"] [dependencies] orchard = { git = "https://github.com/dashpay/orchard.git", rev = "898258d76aab2822249492aede59a02d49278fff", features = ["circuit"] } diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs index 9760f35ef..343390bd8 100644 --- a/grovedb-commitment-tree/src/commitment_tree/mod.rs +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -199,16 +199,14 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { // appended items. A mismatch indicates a partial commit or data // corruption. // - // Exception: with the `test-seeding` feature, the frontier-less seeding - // methods (`append_*_without_frontier`) populate the BulkAppendTree - // while deliberately leaving the frontier empty. Tolerate exactly that - // shape — an empty frontier alongside a populated bulk tree — so seeded - // devnet state can be re-opened. Any other mismatch is still rejected. - let frontier_size = frontier.tree_size(); - if frontier_size != total_count { - let frontier_less_seed = - cfg!(feature = "test-seeding") && frontier_size == 0 && total_count > 0; - if !frontier_less_seed { + // Skipped entirely under `test-seeding-ct`: the frontier-less seeding + // methods (`append_*_without_frontier`) deliberately leave the frontier + // out of sync with the bulk tree, and a seeded tree may then have notes + // added on top, so any `(frontier_size, total_count)` pair must reopen. + #[cfg(not(feature = "test-seeding-ct"))] + { + let frontier_size = frontier.tree_size(); + if frontier_size != total_count { return Err(CommitmentTreeError::InvalidData(format!( "frontier tree_size ({}) != bulk tree total_count ({})", frontier_size, total_count @@ -442,8 +440,8 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { /// /// Mirrors [`CommitmentAppendResult`] but omits `sinsemilla_root`, because the /// Sinsemilla frontier is intentionally left untouched. Only available with the -/// `test-seeding` feature. -#[cfg(feature = "test-seeding")] +/// `test-seeding-ct` feature. +#[cfg(feature = "test-seeding-ct")] #[derive(Debug, Clone)] pub struct FrontierLessAppendResult { /// The BulkAppendTree state root after the append (the Merk child hash). @@ -459,8 +457,8 @@ pub struct FrontierLessAppendResult { /// Summary of a frontier-less bulk seed via /// [`CommitmentTree::append_many_without_frontier`]. /// -/// Only available with the `test-seeding` feature. -#[cfg(feature = "test-seeding")] +/// Only available with the `test-seeding-ct` feature. +#[cfg(feature = "test-seeding-ct")] #[derive(Debug, Clone)] pub struct BulkSeedSummary { /// Number of notes appended during this call. @@ -494,16 +492,19 @@ pub struct BulkSeedSummary { /// still verify, which is exactly what client sync exercises. /// - `cmx` values are **not** validated as Pallas field elements (unlike /// [`append`] and [`append_raw`]), so arbitrary 32-byte filler is accepted. -/// - A tree seeded this way can only be re-[`open`]ed by a build that also -/// enables `test-seeding`, which relaxes the frontier/bulk consistency check -/// for the empty-frontier case. +/// - A seeded tree can only be re-[`open`]ed by a build that also enables +/// `test-seeding-ct`, which drops the frontier/bulk consistency check in +/// [`open`] entirely. After seeding you may keep adding filler this way, or +/// switch to the regular [`append`] / [`append_raw`] to add real, +/// frontier-tracked notes on top (those build the frontier from where it is, +/// i.e. over the post-seed notes only). /// /// [`anchor`]: CommitmentTree::anchor /// [`root_hash`]: CommitmentTree::root_hash /// [`append`]: CommitmentTree::append /// [`append_raw`]: CommitmentTree::append_raw /// [`open`]: CommitmentTree::open -#[cfg(feature = "test-seeding")] +#[cfg(feature = "test-seeding-ct")] impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { /// Append a single note (`cmx || rho || payload`) to the BulkAppendTree /// **without** updating the Sinsemilla frontier. @@ -519,11 +520,11 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { ) -> CostResult { let mut cost = OperationCost::default(); - // Frontier-less seeding only makes sense on an empty frontier. If the - // frontier already has leaves, advancing the bulk tree here would leave - // `frontier_size < total_count`, a mismatch `open` cannot tolerate - // (it only accepts an empty frontier) — i.e. unreopenable state. Reject - // it instead of silently corrupting the tree. + // Frontier-less seeding is meant for a fresh tree whose frontier hasn't + // been built yet — the seeded filler is never reflected in the frontier. + // Reject seeding once the frontier has leaves so filler can only sit at + // the bottom of the tree, never interleaved under real, + // frontier-tracked notes added afterwards. if self.frontier.tree_size() != 0 { return Err(CommitmentTreeError::InvalidData( "frontier-less seeding requires an empty frontier".to_string(), diff --git a/grovedb-commitment-tree/src/commitment_tree/tests.rs b/grovedb-commitment-tree/src/commitment_tree/tests.rs index 4fae209d6..347640c7c 100644 --- a/grovedb-commitment-tree/src/commitment_tree/tests.rs +++ b/grovedb-commitment-tree/src/commitment_tree/tests.rs @@ -968,6 +968,10 @@ mod storage_tests { ); } + // The frontier/bulk consistency check is removed under `test-seeding-ct` + // (frontier-less seeding intentionally desyncs the two), so this test only + // applies when that feature is off. + #[cfg(not(feature = "test-seeding-ct"))] #[test] fn test_open_frontier_total_count_mismatch() { // 1. Create a tree, append 1 item, save @@ -1022,10 +1026,10 @@ mod storage_tests { ); } - // ── Frontier-less seeding (test-seeding feature) ────────────────────── + // ── Frontier-less seeding (test-seeding-ct feature) ────────────────────── /// A correctly-sized DashMemo payload filled with a deterministic pattern. - #[cfg(feature = "test-seeding")] + #[cfg(feature = "test-seeding-ct")] fn seed_payload(index: u8) -> Vec { let mut p = vec![0u8; ciphertext_payload_size::()]; p[0] = index; @@ -1033,7 +1037,7 @@ mod storage_tests { p } - #[cfg(feature = "test-seeding")] + #[cfg(feature = "test-seeding-ct")] #[test] fn test_append_raw_without_frontier_does_not_touch_frontier() { let ctx = MockDataStorageContext::new(); @@ -1057,7 +1061,7 @@ mod storage_tests { ); } - #[cfg(feature = "test-seeding")] + #[cfg(feature = "test-seeding-ct")] #[test] fn test_append_raw_without_frontier_rejects_wrong_payload_size() { let ctx = MockDataStorageContext::new(); @@ -1072,7 +1076,7 @@ mod storage_tests { assert_eq!(ct.total_count(), 0, "tree must not be mutated on rejection"); } - #[cfg(feature = "test-seeding")] + #[cfg(feature = "test-seeding-ct")] #[test] fn test_append_raw_without_frontier_accepts_non_pallas_cmx() { let ctx = MockDataStorageContext::new(); @@ -1089,7 +1093,7 @@ mod storage_tests { assert_eq!(ct.total_count(), 1); } - #[cfg(feature = "test-seeding")] + #[cfg(feature = "test-seeding-ct")] #[test] fn test_append_many_without_frontier_seeds_and_reopens() { let ctx = MockDataStorageContext::new(); @@ -1123,11 +1127,11 @@ mod storage_tests { .expect("state root"); assert_eq!(summary.bulk_state_root, live_root); - // Re-open the seeded (frontier-less) tree: tolerated under test-seeding. + // Re-open the seeded (frontier-less) tree: tolerated under test-seeding-ct. let storage = ct.bulk_tree.dense_tree.storage; let loaded = CommitmentTree::<_, DashMemo>::open(N, TEST_CHUNK_POWER, storage) .value - .expect("open should tolerate empty frontier under test-seeding"); + .expect("open should tolerate empty frontier under test-seeding-ct"); assert_eq!(loaded.total_count(), N, "reopened total_count matches"); assert_eq!(loaded.tree_size(), 0, "reopened frontier still empty"); assert_eq!( @@ -1140,7 +1144,42 @@ mod storage_tests { ); } - #[cfg(feature = "test-seeding")] + #[cfg(feature = "test-seeding-ct")] + #[test] + fn test_add_real_notes_after_frontier_less_seed_then_reopen() { + let ctx = MockDataStorageContext::new(); + let mut ct = + CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); + + // Seed filler frontier-less (frontier stays empty). + const SEEDED: u64 = 6; + let notes = (0..SEEDED).map(|i| (test_leaf(i), test_rho(i as u8), seed_payload(i as u8))); + ct.append_many_without_frontier(notes) + .value + .expect("seed should succeed"); + assert_eq!(ct.tree_size(), 0, "frontier empty after seeding"); + + // Now add a real, frontier-tracked note on top via the normal path. + ct.append(test_leaf(100), test_rho(100), &test_ciphertext(100)) + .value + .expect("normal append on a seeded tree should succeed"); + ct.save().value.expect("save should succeed"); + + let total = ct.total_count(); + assert_eq!(total, SEEDED + 1, "bulk advanced for the real note"); + assert_eq!(ct.tree_size(), 1, "frontier holds only the post-seed note"); + + // Reopen at the mismatched (frontier_size=1, total_count=SEEDED+1): + // tolerated because the consistency check is dropped under the feature. + let storage = ct.bulk_tree.dense_tree.storage; + let loaded = CommitmentTree::<_, DashMemo>::open(total, TEST_CHUNK_POWER, storage) + .value + .expect("reopen of a seeded+appended tree should succeed under test-seeding-ct"); + assert_eq!(loaded.total_count(), total); + assert_eq!(loaded.tree_size(), 1); + } + + #[cfg(feature = "test-seeding-ct")] #[test] fn test_frontier_less_append_rejected_on_non_empty_frontier() { let ctx = MockDataStorageContext::new(); @@ -1178,7 +1217,7 @@ mod storage_tests { ); } - #[cfg(feature = "test-seeding")] + #[cfg(feature = "test-seeding-ct")] #[test] fn test_append_many_without_frontier_empty_input() { let ctx = MockDataStorageContext::new(); diff --git a/grovedb-commitment-tree/src/lib.rs b/grovedb-commitment-tree/src/lib.rs index 74e4cdc75..e09c17a1a 100644 --- a/grovedb-commitment-tree/src/lib.rs +++ b/grovedb-commitment-tree/src/lib.rs @@ -67,7 +67,7 @@ pub use commitment_tree::{ ciphertext_payload_size, deserialize_ciphertext, serialize_ciphertext, CommitmentAppendResult, CommitmentTree, COMMITMENT_TREE_DATA_KEY, }; -#[cfg(feature = "test-seeding")] +#[cfg(feature = "test-seeding-ct")] pub use commitment_tree::{BulkSeedSummary, FrontierLessAppendResult}; pub use error::CommitmentTreeError; #[cfg(feature = "server")] diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index 1abe4be2d..e9e9b2489 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -108,11 +108,11 @@ verify = [ ] estimated_costs = ["full"] zk_client = ["grovedb-commitment-tree", "grovedb-commitment-tree/client"] -# TEST / DEVNET ONLY — forwards `grovedb-commitment-tree/test-seeding`. Exposes -# the frontier-less commitment-tree seeding methods and relaxes the +# TEST / DEVNET ONLY — forwards `grovedb-commitment-tree/test-seeding-ct`. +# Exposes the frontier-less commitment-tree seeding methods and relaxes the # frontier/bulk consistency check in `CommitmentTree::open` so a node can read # seeded (anchor-less) devnet state. Never enable in production. -commitment_tree_test_seeding = ["minimal", "grovedb-commitment-tree/test-seeding"] +test-seeding-ct = ["minimal", "grovedb-commitment-tree/test-seeding-ct"] grovedbg = [ "grovedbg-types", "tokio", From d3adb0283d732311acf71c38615a0170cb2c3b7e Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 25 May 2026 15:24:07 +0700 Subject: [PATCH 06/16] test(commitment-tree): cover frontier-less seeding error paths for patch coverage Add fault-injection to the test storage mock (toggleable get/put failures) and three tests covering the previously-uncovered error branches in the frontier-less seeding methods: the wrapped bulk-append storage error, per-entry error propagation out of the bulk loop, and the empty-input state-root recomputation failure. The remaining commit_mmr flush error is annotated codecov:ignore (unreachable via the seeding API, which always flushes in-call). Raises patch coverage above the 90% gate. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/commitment_tree/mod.rs | 4 + .../src/commitment_tree/tests.rs | 110 ++++++++++++++++++ 2 files changed, 114 insertions(+) diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs index 343390bd8..c7925d834 100644 --- a/grovedb-commitment-tree/src/commitment_tree/mod.rs +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -623,6 +623,10 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { // Flush MMR nodes staged during compaction so the seeded state is // fully persisted; callers don't need a separate commit_mmr(). if let Err(e) = self.commit_mmr() { + // codecov:ignore — commit_mmr only fails on a storage fault during + // the final MMR flush; this method always flushes within the same + // call after appending, so there's no way to leave a pending overlay + // and then fail only the flush via the test mocks. return Err(e).wrap_with_cost(cost); } diff --git a/grovedb-commitment-tree/src/commitment_tree/tests.rs b/grovedb-commitment-tree/src/commitment_tree/tests.rs index 347640c7c..6ccb89087 100644 --- a/grovedb-commitment-tree/src/commitment_tree/tests.rs +++ b/grovedb-commitment-tree/src/commitment_tree/tests.rs @@ -20,14 +20,22 @@ mod storage_tests { /// /// Only `get` and `put` are functional — the rest are stubs /// since `CommitmentTree` only uses data storage operations. + /// + /// `fail_get` / `fail_put` are shared toggles used by the fault-injection + /// tests to make storage reads/writes fail *after* construction, exercising + /// the otherwise-unreachable storage-error branches. struct MockDataStorageContext { data: std::cell::RefCell, Vec>>, + fail_get: std::rc::Rc>, + fail_put: std::rc::Rc>, } impl MockDataStorageContext { fn new() -> Self { Self { data: std::cell::RefCell::new(BTreeMap::new()), + fail_get: Default::default(), + fail_put: Default::default(), } } @@ -37,8 +45,22 @@ mod storage_tests { data.insert(key.to_vec(), value); Self { data: std::cell::RefCell::new(data), + fail_get: Default::default(), + fail_put: Default::default(), } } + + /// Clone the (get, put) failure toggles so a test can flip them after the + /// context has been moved into a `CommitmentTree`. + #[cfg(feature = "test-seeding-ct")] + fn fault_handles( + &self, + ) -> ( + std::rc::Rc>, + std::rc::Rc>, + ) { + (self.fail_get.clone(), self.fail_put.clone()) + } } struct StubBatch; @@ -163,6 +185,15 @@ mod storage_tests { _children_sizes: ChildrenSizesWithIsSumTree, _cost_info: Option, ) -> CostResult<(), grovedb_storage::Error> { + if self.fail_put.get() { + return Err(grovedb_storage::Error::StorageError( + "injected put failure".to_string(), + )) + .wrap_with_cost(OperationCost { + seek_count: 1, + ..Default::default() + }); + } self.data .borrow_mut() .insert(key.as_ref().to_vec(), value.to_vec()); @@ -176,6 +207,15 @@ mod storage_tests { &self, key: K, ) -> CostResult>, grovedb_storage::Error> { + if self.fail_get.get() { + return Err(grovedb_storage::Error::StorageError( + "injected get failure".to_string(), + )) + .wrap_with_cost(OperationCost { + seek_count: 1, + ..Default::default() + }); + } let store = self.data.borrow(); let val = store.get(key.as_ref()).cloned(); let loaded = val.as_ref().map_or(0, |v| v.len() as u64); @@ -1144,6 +1184,76 @@ mod storage_tests { ); } + #[cfg(feature = "test-seeding-ct")] + #[test] + fn test_append_raw_without_frontier_surfaces_bulk_storage_error() { + let ctx = MockDataStorageContext::new(); + let (fail_get, fail_put) = ctx.fault_handles(); + let mut ct = + CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); + + // Make the underlying BulkAppendTree append fail on storage I/O so the + // wrapped "bulk append" error branch is exercised. + fail_get.set(true); + fail_put.set(true); + + let err = ct + .append_raw_without_frontier(test_leaf(0), test_rho(0), &seed_payload(0)) + .value + .expect_err("bulk storage failure should surface"); + assert!( + format!("{}", err).contains("bulk append"), + "error should be wrapped as a bulk append failure: {}", + err + ); + } + + #[cfg(feature = "test-seeding-ct")] + #[test] + fn test_append_many_without_frontier_propagates_entry_error() { + let ctx = MockDataStorageContext::new(); + let mut ct = + CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); + + // A wrong-size payload makes the per-entry append fail; the bulk loop + // must propagate that error rather than swallow it. + let bad_payload = vec![0u8; 1]; + let err = ct + .append_many_without_frontier(std::iter::once((test_leaf(0), test_rho(0), bad_payload))) + .value + .expect_err("a bad entry must propagate out of the bulk loop"); + assert!( + matches!(err, CommitmentTreeError::InvalidPayloadSize { .. }), + "expected the per-entry payload-size error to propagate, got: {}", + err + ); + } + + #[cfg(feature = "test-seeding-ct")] + #[test] + fn test_append_many_without_frontier_surfaces_state_root_error() { + let ctx = MockDataStorageContext::new(); + let (fail_get, _fail_put) = ctx.fault_handles(); + let mut ct = + CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); + + // Seed a few notes so the tree has persisted state to read back. + let notes = (0..3u8).map(|i| (test_leaf(i as u64), test_rho(i), seed_payload(i))); + ct.append_many_without_frontier(notes) + .value + .expect("seed should succeed"); + + // An empty follow-up call recomputes the current state root; with reads + // failing, that recomputation must surface the error. + fail_get.set(true); + let res = + ct.append_many_without_frontier(std::iter::empty::<([u8; 32], [u8; 32], Vec)>()); + assert!( + res.value.is_err(), + "state-root recomputation read failure should surface" + ); + } + #[cfg(feature = "test-seeding-ct")] #[test] fn test_add_real_notes_after_frontier_less_seed_then_reopen() { From 5e4761bac5dd2b730d914d27fe8fbdf55fce3343 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 25 May 2026 18:48:04 +0200 Subject: [PATCH 07/16] perf(bulk-append-tree): cache MMR root so append is O(1), not O(N) per call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `BulkAppendTree::append` called `get_mmr_root()` on every non-compaction append, and `get_mmr_root()` clones the `mmr_overlay` — which accumulates the chunk leaf nodes (each carrying a ~573 KB blob) across all compaction cycles until the session-end flush. Cloning that growing, blob-bearing overlay on every append made bulk seeding O(N^2): a 1M frontier-less seed degraded from ~2400 notes/s to sub-1100 and never finished in reasonable time. The MMR is only mutated on compaction (every `epoch_size` appends), so its root is unchanged in between. Cache it (`last_mmr_root`), refreshing only on compaction. `from_state` keeps it lazy (`None`) because a restored MMR may not be readable until the first append; the first append computes it once, exactly as before. Bulk seeding is now linear at a constant ~2435 notes/s (1M in ~6.85 min vs. 30+ min / non-terminating before). Co-Authored-By: Claude Opus 4.7 (1M context) --- grovedb-bulk-append-tree/src/tree/append.rs | 22 +++++++++++++++++++-- grovedb-bulk-append-tree/src/tree/mod.rs | 15 ++++++++++++++ grovedb-bulk-append-tree/src/tree/tests.rs | 21 ++++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/grovedb-bulk-append-tree/src/tree/append.rs b/grovedb-bulk-append-tree/src/tree/append.rs index 5b7e56626..2cb2debee 100644 --- a/grovedb-bulk-append-tree/src/tree/append.rs +++ b/grovedb-bulk-append-tree/src/tree/append.rs @@ -20,6 +20,8 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { total_count: 0, dense_tree, mmr_overlay: Vec::new(), + // Empty tree → empty MMR → zero root. + last_mmr_root: Some([0u8; 32]), }) } @@ -44,6 +46,9 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { total_count, dense_tree, mmr_overlay: Vec::new(), + // Lazy: the restored MMR may not be readable until an append occurs, + // so don't compute the root here. The first append fills the cache. + last_mmr_root: None, }) } @@ -62,9 +67,20 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { let (compacted, mmr_root, final_dense_root) = match try_result { Some((dense_root, _position)) => { - // Inserted successfully, no compaction needed + // Inserted successfully, no compaction needed. The MMR is + // untouched, so its root is unchanged — use the cached value + // instead of recomputing (which would clone the overlay). On the + // first append after an open the cache is empty, so compute it + // once and store it. hash_count += self.dense_tree.count() as u32 * 2; - let root = self.get_mmr_root()?; + let root = match self.last_mmr_root { + Some(root) => root, + None => { + let root = self.get_mmr_root()?; + self.last_mmr_root = Some(root); + root + } + }; (false, root, dense_root) } None => { @@ -73,6 +89,8 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { // self.mmr_size() reflects the pre-compaction state. let (compact_hashes, mmr_root) = self.compact_with_value(value)?; hash_count += compact_hashes; + // MMR mutated by the compaction — refresh the cached root. + self.last_mmr_root = Some(mmr_root); (true, mmr_root, [0u8; 32]) // empty tree after reset } }; diff --git a/grovedb-bulk-append-tree/src/tree/mod.rs b/grovedb-bulk-append-tree/src/tree/mod.rs index 2fe02f8a9..904c7d2b3 100644 --- a/grovedb-bulk-append-tree/src/tree/mod.rs +++ b/grovedb-bulk-append-tree/src/tree/mod.rs @@ -75,6 +75,21 @@ pub struct BulkAppendTree { /// lifetimes (compaction cycles) so that reads can find recently-pushed /// nodes without a storage round-trip. pub(crate) mmr_overlay: Vec<(u64, Vec)>, + /// Cached MMR root, refreshed only when a compaction mutates the MMR. + /// + /// The MMR is only touched on compaction (every `epoch_size` appends), so + /// its root is unchanged for the ~`epoch_size - 1` appends in between. + /// Caching it avoids recomputing the root — and cloning the (blob-bearing) + /// `mmr_overlay` — on every append, which would otherwise make bulk + /// appends O(N²) as the overlay grows across compaction cycles. + /// + /// `None` means "not yet known" (the state set by [`from_state`], which must + /// stay lazy: the MMR may not be readable until something is appended). It + /// is computed once on the first append after an open, then kept in sync by + /// compaction. + /// + /// [`from_state`]: BulkAppendTree::from_state + pub(crate) last_mmr_root: Option<[u8; 32]>, } impl BulkAppendTree { diff --git a/grovedb-bulk-append-tree/src/tree/tests.rs b/grovedb-bulk-append-tree/src/tree/tests.rs index 65c5d2c5e..3c3f2e490 100644 --- a/grovedb-bulk-append-tree/src/tree/tests.rs +++ b/grovedb-bulk-append-tree/src/tree/tests.rs @@ -39,6 +39,27 @@ fn from_state_invalid_height() { assert!(BulkAppendTree::from_state(0, 17u8, MemStorageContext::new()).is_err()); } +#[test] +fn cached_mmr_root_matches_recomputation_across_compactions() { + // The append fast-path uses a cached MMR root (`last_mmr_root`) instead of + // recomputing it — and cloning the blob-bearing overlay — on every append. + // This guards that the cache never diverges from a fresh recomputation, + // across both compaction and non-compaction appends. + // + // height=2 → epoch_size=4, so 20 appends span 5 compaction cycles. + let mut tree = BulkAppendTree::new(2u8, MemStorageContext::new()).expect("create tree"); + for i in 0..20u8 { + tree.append(&[i]).expect("append"); + let fresh = tree.get_mmr_root().expect("recompute mmr root"); + assert_eq!( + tree.last_mmr_root, + Some(fresh), + "cached MMR root diverged from recomputation after {} append(s)", + i + 1 + ); + } +} + #[test] fn single_append() { let mut tree = BulkAppendTree::new(2u8, MemStorageContext::new()).expect("create tree"); From 60d121900ad3f1b1aa616b81ad60181d1bc417a8 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 25 May 2026 18:48:11 +0200 Subject: [PATCH 08/16] bench(commitment-tree): add 1M frontier-less seeding throughput benchmark Standalone (harness=false) bench that seeds N random filler notes into a real RocksDB-backed CommitmentTree via append_many_without_frontier and reports wall-clock time, splitting compute from the disk commit. Defaults to 1M; override with SEED_N. Gated on test-seeding-ct. cargo bench -p grovedb-commitment-tree --bench seeding --features test-seeding-ct Measured 1M random notes in ~6.85 min (linear ~2435 notes/s) after the MMR-root caching fix. Co-Authored-By: Claude Opus 4.7 (1M context) --- grovedb-commitment-tree/Cargo.toml | 9 ++ grovedb-commitment-tree/benches/seeding.rs | 120 +++++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 grovedb-commitment-tree/benches/seeding.rs diff --git a/grovedb-commitment-tree/Cargo.toml b/grovedb-commitment-tree/Cargo.toml index ec9c734ff..5f7296e00 100644 --- a/grovedb-commitment-tree/Cargo.toml +++ b/grovedb-commitment-tree/Cargo.toml @@ -42,7 +42,16 @@ tempfile = { workspace = true } criterion = { workspace = true } rand = { workspace = true } rand_core = "0.6" +# For the `seeding` benchmark: a real RocksDB-backed storage context. +grovedb-storage = { version = "4.0.0", path = "../storage", features = ["rocksdb_storage"] } +grovedb-path = { version = "4.0.0", path = "../path" } [[bench]] name = "verification" harness = false + +# Frontier-less seeding throughput (test/devnet only). Run with: +# cargo bench -p grovedb-commitment-tree --bench seeding --features test-seeding-ct +[[bench]] +name = "seeding" +harness = false diff --git a/grovedb-commitment-tree/benches/seeding.rs b/grovedb-commitment-tree/benches/seeding.rs new file mode 100644 index 000000000..e289ea631 --- /dev/null +++ b/grovedb-commitment-tree/benches/seeding.rs @@ -0,0 +1,120 @@ +//! Frontier-less seeding throughput benchmark — **test / devnet only**. +//! +//! Pre-populates a real RocksDB-backed [`CommitmentTree`] with `N` random +//! filler notes using [`CommitmentTree::append_many_without_frontier`] (the +//! frontier-less path added for devnet shielded-pool seeding), and reports how +//! long it takes. This is the bulk-append path that skips the per-note +//! Pallas/Sinsemilla hashing, so it measures blake3 BulkAppendTree work + +//! serialization + storage I/O only. +//! +//! Run with: +//! ```text +//! cargo bench -p grovedb-commitment-tree --bench seeding --features test-seeding-ct +//! ``` +//! +//! The note count defaults to 1,000,000 and can be overridden: +//! ```text +//! SEED_N=200000 cargo bench -p grovedb-commitment-tree --bench seeding --features test-seeding-ct +//! ``` + +#[cfg(feature = "test-seeding-ct")] +fn main() { + use std::time::Instant; + + use grovedb_commitment_tree::{ciphertext_payload_size, CommitmentTree, DashMemo}; + use grovedb_path::SubtreePath; + use grovedb_storage::{rocksdb_storage::RocksDbStorage, Storage, StorageBatch}; + use rand::{rngs::StdRng, Rng, SeedableRng}; + + let n: u64 = std::env::var("SEED_N") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(1_000_000); + // Production value: 2^11 = 2048 notes per completed chunk. + let chunk_power: u8 = 11; + let payload_len = ciphertext_payload_size::(); + let entry_len = 64 + payload_len; // cmx(32) + rho(32) + payload + + // Real on-disk RocksDB in a temp dir. + let tmp = tempfile::TempDir::new().expect("create tempdir"); + let storage = + RocksDbStorage::default_rocksdb_with_path(tmp.path()).expect("open rocksdb storage"); + let tx = storage.start_transaction(); + let batch = StorageBatch::new(); + let ct_path = SubtreePath::from(&[b"shielded_notes" as &[u8]]); + let ctx = storage + .get_transactional_storage_context(ct_path, Some(&batch), &tx) + .value; + let mut ct = + CommitmentTree::<_, DashMemo>::new(chunk_power, ctx).expect("create commitment tree"); + + // Deterministic, lazily-generated note stream — never materialized in full. + let mut rng = StdRng::seed_from_u64(0xC0FFEE); + let notes = std::iter::from_fn(move || { + let mut cmx = [0u8; 32]; + let mut rho = [0u8; 32]; + let mut payload = vec![0u8; payload_len]; + rng.fill_bytes(&mut cmx); + rng.fill_bytes(&mut rho); + rng.fill_bytes(&mut payload); + Some((cmx, rho, payload)) + }) + .take(n as usize); + + eprintln!( + "Seeding {n} notes (chunk_power={chunk_power}, payload={payload_len}B, entry={entry_len}B, ~{} MiB of note data)...", + (n as usize * entry_len) / (1024 * 1024) + ); + + // 1. Frontier-less bulk append (compute + in-memory batch accumulation). + let t_seed = Instant::now(); + let summary = ct + .append_many_without_frontier(notes) + .value + .expect("frontier-less seeding"); + let seed_elapsed = t_seed.elapsed(); + + // Release the storage context's borrow of the batch before committing. + drop(ct); + + // 2. Flush the batch into the transaction, then commit the transaction to + // disk (the actual RocksDB write). + let t_commit = Instant::now(); + storage + .commit_multi_context_batch(batch, Some(&tx)) + .value + .expect("commit batch into transaction"); + storage + .commit_transaction(tx) + .value + .expect("commit transaction to disk"); + let commit_elapsed = t_commit.elapsed(); + + let total = seed_elapsed + commit_elapsed; + let rate = |d: std::time::Duration| n as f64 / d.as_secs_f64(); + + eprintln!("---------------------------------------------"); + eprintln!("appended : {}", summary.appended); + eprintln!("total_count : {}", summary.total_count); + eprintln!("compactions : {}", summary.compactions); + eprintln!( + "seed (compute) : {:.3?} ({:.0} notes/s)", + seed_elapsed, + rate(seed_elapsed) + ); + eprintln!("commit (disk) : {:.3?}", commit_elapsed); + eprintln!( + "TOTAL : {:.3?} ({:.0} notes/s)", + total, + rate(total) + ); + eprintln!("---------------------------------------------"); +} + +#[cfg(not(feature = "test-seeding-ct"))] +fn main() { + eprintln!( + "The `seeding` benchmark requires the `test-seeding-ct` feature.\n\ + Run: cargo bench -p grovedb-commitment-tree --bench seeding --features test-seeding-ct" + ); +} From fa7a2306f74b774e55b50383c99f2f816128c399 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 28 May 2026 01:34:54 +0200 Subject: [PATCH 09/16] =?UTF-8?q?feat(commitment-tree):=20batched=20append?= =?UTF-8?q?=5Fmany=5Fraw=20=E2=80=94=20replace=20=5Fwithout=5Ffrontier=20A?= =?UTF-8?q?PI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-leaf `CommitmentTree::append_raw` was slow on bulk seeds not because of Sinsemilla-per-leaf (amortized ~1 hash via the carry chain) but because `CommitmentFrontier::append` called `self.root_hash()` after every insert, walking the depth-32 Sinsemilla path for ~32 hashes per leaf. For 1M leaves that's ~32M extra hashes. The upstream `incrementalmerkletree::Frontier` already separates append (cheap) from root (expensive); our wrapper conflated them. The earlier `_without_frontier` shape side-stepped this by skipping the frontier entirely, producing a Sinsemilla anchor that reflected only a handful of cmx — wallets reconstructing the tree locally got a root the chain had never recorded, and spend proofs failed. This commit fixes the actual problem. Added: * `CommitmentFrontier::append_no_root(cmx)` — upstream `Frontier::append`, no depth-32 walk; carry-chain cost only. * `CommitmentFrontier::root_hash_with_cost()` — pure accessor that attributes the deferred 32-hash depth walk for batched callers. * `BulkAppendTree::append_no_state_root(value)` — `append` minus the per-leaf `compute_state_root` blake3. `append` now delegates to it + one final `compute_current_state_root`. * `BulkAppendTree::append_many>>` — batched bulk-tree appends, one state-root computation at the end. * `CommitmentTree::append_many_raw)>>` — byte-for-byte equivalent to N × `append_raw` (same dense-buffer, MMR, `CommitmentFrontier::serialize()`, final state and Sinsemilla roots) but the Sinsemilla anchor and bulk state root are each computed exactly once at the end. Flushes the MMR overlay before return. * `compute_current_state_root` now uses the `last_mmr_root` cache when populated (O(1) on the hot path). Removed: * `test-seeding-ct` Cargo feature in `grovedb-commitment-tree` and the forwarding feature in `grovedb`. * `append_raw_without_frontier`, `append_many_without_frontier`, `FrontierLessAppendResult`, `BulkSeedSummary`, and their lib.rs re-exports. * The `#[cfg(not(feature = "test-seeding-ct"))]` gating around the frontier/bulk consistency check in `CommitmentTree::open` — the check is now unconditional. * Fault-injection mock state and storage-fault tests that only existed to cover the deleted branches. Tests: * `append_many_raw_byte_for_byte_matches_per_leaf` — for N ∈ {0,1,2,3,100, 2048,10_000}, the per-leaf and batched paths produce identical `frontier.root_hash()`, `CommitmentFrontier::serialize()`, bulk state root, and `total_count`. * `append_many_raw_anchor_is_spend_usable` — independent Sinsemilla auth-path recomputation; verifies `orchard::MerklePath::from_parts(pos, path).root(cmx) == ct.anchor()`. * `append_no_root_cost_omits_per_leaf_depth_walk` — confirms the savings are exactly the per-leaf depth walks. Co-Authored-By: Claude Opus 4.7 (1M context) --- grovedb-bulk-append-tree/src/tree/append.rs | 126 ++++- grovedb-bulk-append-tree/src/tree/mod.rs | 36 ++ grovedb-commitment-tree/Cargo.toml | 15 +- grovedb-commitment-tree/benches/seeding.rs | 55 +- .../src/commitment_frontier/mod.rs | 51 ++ .../src/commitment_tree/mod.rs | 395 ++++++-------- .../src/commitment_tree/tests.rs | 489 ++++++++---------- grovedb-commitment-tree/src/lib.rs | 2 - grovedb/Cargo.toml | 5 - 9 files changed, 585 insertions(+), 589 deletions(-) diff --git a/grovedb-bulk-append-tree/src/tree/append.rs b/grovedb-bulk-append-tree/src/tree/append.rs index 2cb2debee..9ad0fc54e 100644 --- a/grovedb-bulk-append-tree/src/tree/append.rs +++ b/grovedb-bulk-append-tree/src/tree/append.rs @@ -5,7 +5,10 @@ use grovedb_merkle_mountain_range::{ }; use grovedb_storage::StorageContext; -use super::{capacity_for_height, hash::compute_state_root, AppendResult, BulkAppendTree}; +use super::{ + capacity_for_height, hash::compute_state_root, AppendManyResult, AppendNoStateRootResult, + AppendResult, BulkAppendTree, +}; use crate::{chunk::serialize_chunk_blob, BulkAppendError}; impl<'db, S: StorageContext<'db>> BulkAppendTree { @@ -55,33 +58,54 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// Append a value to the tree. /// /// Handles dense tree insert, auto-compaction when the buffer fills, and - /// state root computation. + /// state root computation. For batched inserts prefer + /// [`append_many`](Self::append_many) or [`append_no_state_root`](Self::append_no_state_root) + /// — they skip the per-leaf state-root blake3 call. pub fn append(&mut self, value: &[u8]) -> Result { + let r = self.append_no_state_root(value)?; + let state_root = self.compute_current_state_root()?; + Ok(AppendResult { + state_root, + global_position: r.global_position, + // +1 for the blake3 state-root computation we just did. + hash_count: r.hash_count.saturating_add(1), + compacted: r.compacted, + }) + } + + /// Append a value without computing the per-leaf state root. + /// + /// Equivalent to [`append`](Self::append) minus the final + /// `compute_state_root` blake3 hash. Use inside a batch (typically via + /// [`append_many`](Self::append_many) or + /// [`CommitmentTree::append_many_raw`]) and recover the state root once at + /// the end via [`compute_current_state_root`](Self::compute_current_state_root). + /// Storage mutation is identical to [`append`](Self::append). + /// + /// [`CommitmentTree::append_many_raw`]: ../../grovedb_commitment_tree/struct.CommitmentTree.html#method.append_many_raw + pub fn append_no_state_root( + &mut self, + value: &[u8], + ) -> Result { let mut hash_count: u32 = 0; let global_position = self.total_count; - // 1. Try to insert into the dense tree buffer + // 1. Try to insert into the dense tree buffer. let try_result = self.dense_tree.try_insert(value).unwrap().map_err(|e| { BulkAppendError::StorageError(format!("dense tree insert failed: {}", e)) })?; - let (compacted, mmr_root, final_dense_root) = match try_result { - Some((dense_root, _position)) => { + let compacted = match try_result { + Some((_dense_root, _position)) => { // Inserted successfully, no compaction needed. The MMR is - // untouched, so its root is unchanged — use the cached value - // instead of recomputing (which would clone the overlay). On the - // first append after an open the cache is empty, so compute it - // once and store it. + // untouched, so its root is unchanged — keep the cache as-is. + // (On the very first append after a lazy open the cache is + // `None`; we don't seed it here because we don't need it — + // the caller will recover the state root via + // `compute_current_state_root` at the end of the batch, which + // populates the cache then.) hash_count += self.dense_tree.count() as u32 * 2; - let root = match self.last_mmr_root { - Some(root) => root, - None => { - let root = self.get_mmr_root()?; - self.last_mmr_root = Some(root); - root - } - }; - (false, root, dense_root) + false } None => { // Dense tree is full — compact existing entries + new value. @@ -91,27 +115,77 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { hash_count += compact_hashes; // MMR mutated by the compaction — refresh the cached root. self.last_mmr_root = Some(mmr_root); - (true, mmr_root, [0u8; 32]) // empty tree after reset + true } }; self.total_count += 1; - // 2. Compute state root (+1 hash) - let state_root = compute_state_root(&mmr_root, &final_dense_root); - hash_count += 1; - - Ok(AppendResult { - state_root, + Ok(AppendNoStateRootResult { global_position, hash_count, compacted, }) } + /// Batch-append a sequence of values, computing the state root **once** at + /// the end instead of once per leaf. + /// + /// Byte-for-byte equivalent to calling [`append`](Self::append) per value + /// (same dense-tree state, same MMR, same final `state_root`), but skips + /// the per-leaf blake3 state-root computation. Used by + /// [`CommitmentTree::append_many_raw`] to keep large bulk seeds linear. + /// + /// On error the tree is left partially mutated — discard the surrounding + /// transaction if you need all-or-nothing semantics. + /// + /// [`CommitmentTree::append_many_raw`]: ../../grovedb_commitment_tree/struct.CommitmentTree.html#method.append_many_raw + pub fn append_many(&mut self, values: I) -> Result + where + I: IntoIterator>, + { + let mut appended: u64 = 0; + let mut hash_count: u64 = 0; + let mut compactions: u64 = 0; + let mut last_global_position: Option = None; + + for value in values { + let r = self.append_no_state_root(&value)?; + last_global_position = Some(r.global_position); + hash_count = hash_count.saturating_add(u64::from(r.hash_count)); + if r.compacted { + compactions += 1; + } + appended += 1; + } + + // One blake3 for the final state root (matching the +1 each per-leaf + // `append` would have counted). + let state_root = self.compute_current_state_root()?; + if appended > 0 { + hash_count = hash_count.saturating_add(1); + } + + Ok(AppendManyResult { + state_root, + last_global_position, + appended, + hash_count, + compactions, + }) + } + /// Compute the current state root without modifying the tree. + /// + /// Uses the cached MMR root when available, so this is O(1) on the + /// post-first-append fast path (no overlay clone). Falls back to a one-shot + /// `get_mmr_root` only when the cache is empty (e.g. immediately after a + /// lazy `from_state` with no appends yet). pub fn compute_current_state_root(&self) -> Result<[u8; 32], BulkAppendError> { - let mmr_root = self.get_mmr_root()?; + let mmr_root = match self.last_mmr_root { + Some(r) => r, + None => self.get_mmr_root()?, + }; let dense_root = self.dense_tree.root_hash().unwrap().map_err(|e| { BulkAppendError::StorageError(format!("dense tree root_hash failed: {}", e)) })?; diff --git a/grovedb-bulk-append-tree/src/tree/mod.rs b/grovedb-bulk-append-tree/src/tree/mod.rs index 904c7d2b3..16e7a0f3c 100644 --- a/grovedb-bulk-append-tree/src/tree/mod.rs +++ b/grovedb-bulk-append-tree/src/tree/mod.rs @@ -40,6 +40,42 @@ pub struct AppendResult { pub compacted: bool, } +/// Result returned by [`BulkAppendTree::append_no_state_root`]. +/// +/// Same as [`AppendResult`] minus the state root, which the caller computes +/// once at the end of a batch via +/// [`BulkAppendTree::compute_current_state_root`]. +#[cfg(feature = "storage")] +#[derive(Debug, Clone, Copy)] +pub struct AppendNoStateRootResult { + /// The 0-based global position of the appended value. + pub global_position: u64, + /// Number of blake3 hash calls performed during this append (excludes the + /// deferred state-root computation). + pub hash_count: u32, + /// Whether compaction (epoch flush) occurred. + pub compacted: bool, +} + +/// Result returned by [`BulkAppendTree::append_many`]. +#[cfg(feature = "storage")] +#[derive(Debug, Clone)] +pub struct AppendManyResult { + /// State root after all appends. Equal to what the final per-leaf + /// [`AppendResult::state_root`] would have been. + pub state_root: [u8; 32], + /// 0-based global position of the last appended value, or `None` if the + /// input iterator was empty. + pub last_global_position: Option, + /// Number of values actually appended (the iterator's length). + pub appended: u64, + /// Sum of blake3 hashes across the batch, including the single end-of-batch + /// state-root computation. + pub hash_count: u64, + /// Number of compactions (epoch flushes) that occurred during the batch. + pub compactions: u64, +} + /// Compute MMR size from leaf count: `2 * n - popcount(n)`. /// /// This is a well-known MMR property: the total number of nodes (leaves + diff --git a/grovedb-commitment-tree/Cargo.toml b/grovedb-commitment-tree/Cargo.toml index 5f7296e00..2358e59a7 100644 --- a/grovedb-commitment-tree/Cargo.toml +++ b/grovedb-commitment-tree/Cargo.toml @@ -15,16 +15,6 @@ default = [] server = ["grovedb-storage", "grovedb-bulk-append-tree/storage"] client = ["shardtree"] sqlite = ["client", "rusqlite"] -# TEST / DEVNET ONLY — do not enable in production. -# -# Adds frontier-less seeding methods (`append_raw_without_frontier`, -# `append_many_without_frontier`) that populate the underlying BulkAppendTree -# WITHOUT updating the Sinsemilla frontier. This lets benchmarks pre-populate a -# shielded pool with a large N of filler notes at blake3 speed, skipping the -# per-note Pallas/Sinsemilla hashing. The resulting tree has NO valid Orchard -# anchor, so seeded notes are not spendable; chunk proofs (authenticated by the -# blake3 bulk state root) still verify, which is what client sync exercises. -test-seeding-ct = ["server"] [dependencies] orchard = { git = "https://github.com/dashpay/orchard.git", rev = "898258d76aab2822249492aede59a02d49278fff", features = ["circuit"] } @@ -50,8 +40,9 @@ grovedb-path = { version = "4.0.0", path = "../path" } name = "verification" harness = false -# Frontier-less seeding throughput (test/devnet only). Run with: -# cargo bench -p grovedb-commitment-tree --bench seeding --features test-seeding-ct +# Batched commitment-tree seeding throughput. Run with: +# cargo bench -p grovedb-commitment-tree --bench seeding --features server [[bench]] name = "seeding" harness = false +required-features = ["server"] diff --git a/grovedb-commitment-tree/benches/seeding.rs b/grovedb-commitment-tree/benches/seeding.rs index e289ea631..b90542760 100644 --- a/grovedb-commitment-tree/benches/seeding.rs +++ b/grovedb-commitment-tree/benches/seeding.rs @@ -1,27 +1,29 @@ -//! Frontier-less seeding throughput benchmark — **test / devnet only**. +//! Batched commitment-tree seeding throughput benchmark. //! //! Pre-populates a real RocksDB-backed [`CommitmentTree`] with `N` random -//! filler notes using [`CommitmentTree::append_many_without_frontier`] (the -//! frontier-less path added for devnet shielded-pool seeding), and reports how -//! long it takes. This is the bulk-append path that skips the per-note -//! Pallas/Sinsemilla hashing, so it measures blake3 BulkAppendTree work + -//! serialization + storage I/O only. +//! notes via [`CommitmentTree::append_many_raw`] — the batched API that +//! computes the Sinsemilla anchor and the BulkAppendTree state root once at +//! the end of the batch instead of once per leaf. Byte-for-byte equivalent to +//! `N × CommitmentTree::append_raw`, just without the per-leaf depth-32 +//! Sinsemilla root walk. //! //! Run with: //! ```text -//! cargo bench -p grovedb-commitment-tree --bench seeding --features test-seeding-ct +//! cargo bench -p grovedb-commitment-tree --bench seeding //! ``` //! //! The note count defaults to 1,000,000 and can be overridden: //! ```text -//! SEED_N=200000 cargo bench -p grovedb-commitment-tree --bench seeding --features test-seeding-ct +//! SEED_N=200000 cargo bench -p grovedb-commitment-tree --bench seeding //! ``` -#[cfg(feature = "test-seeding-ct")] +#[cfg(feature = "server")] fn main() { use std::time::Instant; - use grovedb_commitment_tree::{ciphertext_payload_size, CommitmentTree, DashMemo}; + use grovedb_commitment_tree::{ + ciphertext_payload_size, merkle_hash_from_bytes, CommitmentTree, DashMemo, + }; use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::RocksDbStorage, Storage, StorageBatch}; use rand::{rngs::StdRng, Rng, SeedableRng}; @@ -49,12 +51,19 @@ fn main() { CommitmentTree::<_, DashMemo>::new(chunk_power, ctx).expect("create commitment tree"); // Deterministic, lazily-generated note stream — never materialized in full. + // The cmx is rejection-sampled to a valid Pallas field element so the + // batched anchor stays sound (matching what real notes do). let mut rng = StdRng::seed_from_u64(0xC0FFEE); let notes = std::iter::from_fn(move || { let mut cmx = [0u8; 32]; + loop { + rng.fill_bytes(&mut cmx); + if merkle_hash_from_bytes(&cmx).is_some() { + break; + } + } let mut rho = [0u8; 32]; let mut payload = vec![0u8; payload_len]; - rng.fill_bytes(&mut cmx); rng.fill_bytes(&mut rho); rng.fill_bytes(&mut payload); Some((cmx, rho, payload)) @@ -62,16 +71,18 @@ fn main() { .take(n as usize); eprintln!( - "Seeding {n} notes (chunk_power={chunk_power}, payload={payload_len}B, entry={entry_len}B, ~{} MiB of note data)...", + "Seeding {n} notes via append_many_raw (chunk_power={chunk_power}, payload={payload_len}B, entry={entry_len}B, ~{} MiB of note data)...", (n as usize * entry_len) / (1024 * 1024) ); - // 1. Frontier-less bulk append (compute + in-memory batch accumulation). + // 1. Batched bulk append — compute + in-memory batch accumulation. + // Sinsemilla anchor + bulk state root are each computed exactly once + // at the end, not per leaf. let t_seed = Instant::now(); - let summary = ct - .append_many_without_frontier(notes) + let result = ct + .append_many_raw(notes) .value - .expect("frontier-less seeding"); + .expect("batched commitment-tree seeding"); let seed_elapsed = t_seed.elapsed(); // Release the storage context's borrow of the batch before committing. @@ -94,9 +105,9 @@ fn main() { let rate = |d: std::time::Duration| n as f64 / d.as_secs_f64(); eprintln!("---------------------------------------------"); - eprintln!("appended : {}", summary.appended); - eprintln!("total_count : {}", summary.total_count); - eprintln!("compactions : {}", summary.compactions); + eprintln!("appended : {}", n); + eprintln!("last position : {}", result.global_position); + eprintln!("compacted : {}", result.compacted); eprintln!( "seed (compute) : {:.3?} ({:.0} notes/s)", seed_elapsed, @@ -111,10 +122,10 @@ fn main() { eprintln!("---------------------------------------------"); } -#[cfg(not(feature = "test-seeding-ct"))] +#[cfg(not(feature = "server"))] fn main() { eprintln!( - "The `seeding` benchmark requires the `test-seeding-ct` feature.\n\ - Run: cargo bench -p grovedb-commitment-tree --bench seeding --features test-seeding-ct" + "The `seeding` benchmark requires the `server` feature.\n\ + Run: cargo bench -p grovedb-commitment-tree --bench seeding --features server" ); } diff --git a/grovedb-commitment-tree/src/commitment_frontier/mod.rs b/grovedb-commitment-tree/src/commitment_frontier/mod.rs index fae356304..80adcadac 100644 --- a/grovedb-commitment-tree/src/commitment_frontier/mod.rs +++ b/grovedb-commitment-tree/src/commitment_frontier/mod.rs @@ -43,6 +43,10 @@ impl CommitmentFrontier { /// Returns the new Sinsemilla root hash after the append. The returned /// [`OperationCost`] tracks `sinsemilla_hash_calls`: 32 hashes for the /// leaf-to-root path plus `trailing_ones(position)` ommer hashes. + /// + /// Prefer [`append_no_root`](Self::append_no_root) inside a batch — it + /// skips the per-leaf depth-32 root walk, which dominates the cost of + /// large bulk inserts. pub fn append(&mut self, cmx: [u8; 32]) -> CostResult<[u8; 32], CommitmentTreeError> { let mut cost = OperationCost::default(); let leaf = match merkle_hash_from_bytes(&cmx) { @@ -67,6 +71,40 @@ impl CommitmentFrontier { Ok(self.root_hash()).wrap_with_cost(cost) } + /// Append a commitment without recomputing the root. + /// + /// Cheaper than [`append`](Self::append) for batched inserts: defers the + /// depth-32 Sinsemilla root walk. The caller must call + /// [`root_hash`](Self::root_hash) (or [`anchor`](Self::anchor)) once after + /// the batch to get the post-batch state; intermediate roots are not + /// produced. + /// + /// Cost: amortized ~1 Sinsemilla hash per leaf (the carry chain inside + /// `Frontier::append`) vs ~33 for [`append`](Self::append). + pub fn append_no_root(&mut self, cmx: [u8; 32]) -> CostResult<(), CommitmentTreeError> { + let mut cost = OperationCost::default(); + let leaf = match merkle_hash_from_bytes(&cmx) { + Some(l) => l, + None => { + return Err(CommitmentTreeError::InvalidFieldElement).wrap_with_cost(cost); + } + }; + + // Count only the carry-chain hashes performed by `Frontier::append` + // itself — the depth-32 root walk is *not* done here. + let ommer_hashes = self + .frontier + .value() + .map(|f| u64::from(f.position()).trailing_ones()) + .unwrap_or(0); + cost.sinsemilla_hash_calls += ommer_hashes; + + if !self.frontier.append(leaf) { + return Err(CommitmentTreeError::TreeFull).wrap_with_cost(cost); + } + Ok(()).wrap_with_cost(cost) + } + /// Get the current Sinsemilla root hash as 32 bytes. /// /// Returns the empty tree root if no leaves have been appended. @@ -74,6 +112,19 @@ impl CommitmentFrontier { self.frontier.root().to_bytes() } + /// Same as [`root_hash`](Self::root_hash) but attributes the depth-32 + /// Sinsemilla walk to the returned [`OperationCost`]. Used by batched + /// callers (e.g. [`CommitmentTree::append_many_raw`]) so the deferred + /// per-leaf cost is recovered at the single end-of-batch root computation. + /// + /// [`CommitmentTree::append_many_raw`]: crate::CommitmentTree::append_many_raw + pub fn root_hash_with_cost(&self) -> grovedb_costs::CostContext<[u8; 32]> { + let mut cost = OperationCost::default(); + // The Frontier walks `FRONTIER_DEPTH` levels to derive the root. + cost.sinsemilla_hash_calls += FRONTIER_DEPTH as u32; + self.root_hash().wrap_with_cost(cost) + } + /// Get the current root as an Orchard `Anchor`. pub fn anchor(&self) -> Anchor { Anchor::from(self.frontier.root()) diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs index c7925d834..2208bbaa1 100644 --- a/grovedb-commitment-tree/src/commitment_tree/mod.rs +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -197,22 +197,15 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { // Validate that the frontier and bulk tree agree on the number of // appended items. A mismatch indicates a partial commit or data - // corruption. - // - // Skipped entirely under `test-seeding-ct`: the frontier-less seeding - // methods (`append_*_without_frontier`) deliberately leave the frontier - // out of sync with the bulk tree, and a seeded tree may then have notes - // added on top, so any `(frontier_size, total_count)` pair must reopen. - #[cfg(not(feature = "test-seeding-ct"))] - { - let frontier_size = frontier.tree_size(); - if frontier_size != total_count { - return Err(CommitmentTreeError::InvalidData(format!( - "frontier tree_size ({}) != bulk tree total_count ({})", - frontier_size, total_count - ))) - .wrap_with_cost(cost); - } + // corruption; both [`append_raw`] and [`append_many_raw`] keep the two + // in sync. + let frontier_size = frontier.tree_size(); + if frontier_size != total_count { + return Err(CommitmentTreeError::InvalidData(format!( + "frontier tree_size ({}) != bulk tree total_count ({})", + frontier_size, total_count + ))) + .wrap_with_cost(cost); } Ok(Self { @@ -330,6 +323,156 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { .wrap_with_cost(cost) } + /// Batch-append a sequence of `(cmx, rho, payload)` entries. + /// + /// Byte-for-byte equivalent to calling [`append_raw`](Self::append_raw) + /// once per entry — same dense-buffer state, same chunk MMR, same + /// `CommitmentFrontier` serialization, same final `bulk_state_root` and + /// `sinsemilla_root` — but computes the Sinsemilla anchor and the + /// BulkAppendTree state root **exactly once at the end** instead of once + /// per leaf. + /// + /// # Why batch + /// + /// Per-leaf [`append_raw`](Self::append_raw) walks the full depth-32 + /// Sinsemilla path to derive a fresh anchor on every call — ~32 + /// Sinsemilla hashes per leaf, ~33× the actual carry-chain work the + /// upstream `Frontier::append` performs internally. For 1M leaves that + /// dominates everything. Batching defers the depth walk to one final + /// `root_hash` call. + /// + /// # Returns + /// + /// A [`CommitmentAppendResult`] shaped like what the **final** per-leaf + /// [`append_raw`](Self::append_raw) would have returned: `sinsemilla_root` + /// and `bulk_state_root` are the post-batch values; `global_position` is + /// the position of the last appended entry; `hash_count` is the sum across + /// the batch (including the one final state-root blake3); `compacted` is + /// `true` if any compaction occurred during the batch. + /// + /// # Atomicity + /// + /// On error (e.g. invalid cmx or payload size in the middle of the input) + /// any entries already processed remain in the tree — discard the + /// surrounding transaction if you need all-or-nothing semantics. This + /// matches the per-leaf behavior of calling [`append_raw`](Self::append_raw) + /// in a loop. + /// + /// Call [`save`](Self::save) afterwards to persist the updated frontier. + pub fn append_many_raw( + &mut self, + entries: I, + ) -> CostResult + where + I: IntoIterator)>, + { + let mut cost = OperationCost::default(); + let expected_payload = ciphertext_payload_size::(); + + let mut appended: u64 = 0; + let mut hash_count: u32 = 0; + let mut any_compacted = false; + // Track the last appended position. If the input is empty we fall back + // to the tree's current top of range (or 0 if empty) — the byte-for-byte + // contract only governs frontier+bulk state, not this field for N=0. + let starting_total = self.bulk_tree.total_count; + let mut last_global_position: u64 = starting_total.saturating_sub(1); + + for (cmx, rho, payload) in entries { + // Pre-validate cmx (Pallas field element) and payload size *before* + // any mutation for this entry — mirrors append_raw's ordering so a + // bad entry doesn't leave a half-written row behind. + if crate::commitment_frontier::merkle_hash_from_bytes(&cmx).is_none() { + return Err(CommitmentTreeError::InvalidFieldElement).wrap_with_cost(cost); + } + if payload.len() != expected_payload { + return Err(CommitmentTreeError::InvalidPayloadSize { + expected: expected_payload, + actual: payload.len(), + }) + .wrap_with_cost(cost); + } + + // 1. Build cmx||rho||payload and append to BulkAppendTree, deferring + // the per-leaf state_root blake3. + let mut item_value = Vec::with_capacity(64 + payload.len()); + item_value.extend_from_slice(&cmx); + item_value.extend_from_slice(&rho); + item_value.extend_from_slice(&payload); + + let r = match self.bulk_tree.append_no_state_root(&item_value) { + Ok(r) => r, + // codecov:ignore — only reachable on a storage fault during the + // dense-tree insert or MMR compaction (see `append_raw`'s + // sibling branch for the full rationale). + Err(e) => { + return Err(CommitmentTreeError::InvalidData(format!( + "bulk append: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + hash_count = hash_count.saturating_add(r.hash_count); + if r.compacted { + any_compacted = true; + } + last_global_position = r.global_position; + + // 2. Append cmx to the Sinsemilla frontier, deferring the depth-32 + // root walk. Validation here is now redundant with the pre-check + // above but is cheap and keeps the cost accounting correct. + if let Err(e) = self.frontier.append_no_root(cmx).unwrap_add_cost(&mut cost) { + return Err(e).wrap_with_cost(cost); + } + + appended += 1; + } + + // End-of-batch: pay the deferred costs **once**. + // * `compute_current_state_root` runs one blake3 (matching the +1 each + // per-leaf `append` would have added). + // * `root_hash_with_cost` runs the depth-32 Sinsemilla walk and + // attributes its sinsemilla_hash_calls to `cost`. + let bulk_state_root = match self.bulk_tree.compute_current_state_root() { + Ok(r) => r, + // codecov:ignore — only reachable on a storage fault (same as the + // per-entry bulk-append failure above). + Err(e) => { + return Err(CommitmentTreeError::InvalidData(format!( + "state root: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + if appended > 0 { + hash_count = hash_count.saturating_add(1); + } + + let root_ctx = self.frontier.root_hash_with_cost(); + cost += root_ctx.cost; + let sinsemilla_root = root_ctx.value; + + // Flush MMR nodes staged during compaction so the seeded state is fully + // persisted; callers don't need a separate `commit_mmr` after this. + if let Err(e) = self.commit_mmr() { + // codecov:ignore — only reachable on a storage fault during the + // final MMR flush; the bench / seed path can't isolate that fault + // from the appends with our test mocks. + return Err(e).wrap_with_cost(cost); + } + + Ok(CommitmentAppendResult { + sinsemilla_root, + bulk_state_root, + global_position: last_global_position, + hash_count, + compacted: any_compacted, + }) + .wrap_with_cost(cost) + } + /// Persist the current frontier state to storage. pub fn save(&self) -> CostResult<(), CommitmentTreeError> { let mut cost = OperationCost::default(); @@ -434,223 +577,3 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { self.bulk_tree.chunk_count() } } - -/// Result of a single frontier-less append via -/// [`CommitmentTree::append_raw_without_frontier`]. -/// -/// Mirrors [`CommitmentAppendResult`] but omits `sinsemilla_root`, because the -/// Sinsemilla frontier is intentionally left untouched. Only available with the -/// `test-seeding-ct` feature. -#[cfg(feature = "test-seeding-ct")] -#[derive(Debug, Clone)] -pub struct FrontierLessAppendResult { - /// The BulkAppendTree state root after the append (the Merk child hash). - pub bulk_state_root: [u8; 32], - /// The 0-based global position of the appended value. - pub global_position: u64, - /// Number of blake3 hash calls performed during the bulk append. - pub hash_count: u32, - /// Whether compaction (epoch flush) occurred during this append. - pub compacted: bool, -} - -/// Summary of a frontier-less bulk seed via -/// [`CommitmentTree::append_many_without_frontier`]. -/// -/// Only available with the `test-seeding-ct` feature. -#[cfg(feature = "test-seeding-ct")] -#[derive(Debug, Clone)] -pub struct BulkSeedSummary { - /// Number of notes appended during this call. - pub appended: u64, - /// The tree's total item count after the call. - pub total_count: u64, - /// The final BulkAppendTree state root (the Merk child hash). This binds - /// only the bulk data; a frontier-less seeded tree has no valid Orchard - /// anchor. - pub bulk_state_root: [u8; 32], - /// Total blake3 hash calls performed across all appends and compactions. - pub hash_count: u64, - /// Number of epoch compactions (chunk finalizations) triggered. - pub compactions: u64, -} - -/// Frontier-less seeding API — **test / devnet only**. -/// -/// These methods append note entries to the underlying [`BulkAppendTree`] -/// WITHOUT updating the Sinsemilla [`CommitmentFrontier`]. They exist to -/// pre-populate the shielded pool with a large number of filler notes for -/// sync/scale benchmarking without paying the per-note Pallas/Sinsemilla -/// hashing cost (or the full Drive insert path). -/// -/// # Consequences -/// -/// - The tree has **no valid Orchard anchor** afterwards: [`anchor`] and -/// [`root_hash`] reflect an empty frontier, so notes seeded this way are -/// **not spendable**. The BulkAppendTree chunk proofs — authenticated by the -/// blake3 `bulk_state_root` rather than the frontier — are unaffected and -/// still verify, which is exactly what client sync exercises. -/// - `cmx` values are **not** validated as Pallas field elements (unlike -/// [`append`] and [`append_raw`]), so arbitrary 32-byte filler is accepted. -/// - A seeded tree can only be re-[`open`]ed by a build that also enables -/// `test-seeding-ct`, which drops the frontier/bulk consistency check in -/// [`open`] entirely. After seeding you may keep adding filler this way, or -/// switch to the regular [`append`] / [`append_raw`] to add real, -/// frontier-tracked notes on top (those build the frontier from where it is, -/// i.e. over the post-seed notes only). -/// -/// [`anchor`]: CommitmentTree::anchor -/// [`root_hash`]: CommitmentTree::root_hash -/// [`append`]: CommitmentTree::append -/// [`append_raw`]: CommitmentTree::append_raw -/// [`open`]: CommitmentTree::open -#[cfg(feature = "test-seeding-ct")] -impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { - /// Append a single note (`cmx || rho || payload`) to the BulkAppendTree - /// **without** updating the Sinsemilla frontier. - /// - /// The payload length must equal [`ciphertext_payload_size::()`]. The - /// `cmx` is stored verbatim and is *not* validated as a Pallas field - /// element. See the impl-level docs for the consequences. - pub fn append_raw_without_frontier( - &mut self, - cmx: [u8; 32], - rho: [u8; 32], - payload: &[u8], - ) -> CostResult { - let mut cost = OperationCost::default(); - - // Frontier-less seeding is meant for a fresh tree whose frontier hasn't - // been built yet — the seeded filler is never reflected in the frontier. - // Reject seeding once the frontier has leaves so filler can only sit at - // the bottom of the tree, never interleaved under real, - // frontier-tracked notes added afterwards. - if self.frontier.tree_size() != 0 { - return Err(CommitmentTreeError::InvalidData( - "frontier-less seeding requires an empty frontier".to_string(), - )) - .wrap_with_cost(cost); - } - - // Validate payload size — kept because it keeps stored entries - // well-formed for chunk proofs and client deserialization. (cmx is - // intentionally not validated as a Pallas field element here.) - let expected = ciphertext_payload_size::(); - if payload.len() != expected { - return Err(CommitmentTreeError::InvalidPayloadSize { - expected, - actual: payload.len(), - }) - .wrap_with_cost(cost); - } - - let mut item_value = Vec::with_capacity(64 + payload.len()); - item_value.extend_from_slice(&cmx); - item_value.extend_from_slice(&rho); - item_value.extend_from_slice(payload); - - let bulk_result = match self.bulk_tree.append(&item_value) { - Ok(r) => r, - Err(e) => { - return Err(CommitmentTreeError::InvalidData(format!( - "bulk append: {}", - e - ))) - .wrap_with_cost(cost); - } - }; - cost.hash_node_calls += bulk_result.hash_count; - - Ok(FrontierLessAppendResult { - bulk_state_root: bulk_result.state_root, - global_position: bulk_result.global_position, - hash_count: bulk_result.hash_count, - compacted: bulk_result.compacted, - }) - .wrap_with_cost(cost) - } - - /// Bulk-append many notes to the BulkAppendTree **without** updating the - /// Sinsemilla frontier. - /// - /// Each item is `(cmx, rho, payload)`. The iterator is consumed lazily, so - /// a generator can stream a large `N` without materializing it. MMR nodes - /// buffered during compaction are flushed via [`commit_mmr`] before - /// returning, so the caller does not need a separate `commit_mmr()` call. - /// The frontier is left untouched; callers typically set the parent - /// `Element::CommitmentTree(total_count, ..)` from the returned summary. - /// - /// [`commit_mmr`]: CommitmentTree::commit_mmr - pub fn append_many_without_frontier( - &mut self, - notes: I, - ) -> CostResult - where - I: IntoIterator)>, - { - let mut cost = OperationCost::default(); - - // Fail fast before consuming the iterator: frontier-less seeding is only - // valid on an empty frontier (see `append_raw_without_frontier`). - if self.frontier.tree_size() != 0 { - return Err(CommitmentTreeError::InvalidData( - "frontier-less seeding requires an empty frontier".to_string(), - )) - .wrap_with_cost(cost); - } - - let mut appended: u64 = 0; - let mut hash_count: u64 = 0; - let mut compactions: u64 = 0; - let mut bulk_state_root = [0u8; 32]; - - for (cmx, rho, payload) in notes { - let r = match self - .append_raw_without_frontier(cmx, rho, &payload) - .unwrap_add_cost(&mut cost) - { - Ok(r) => r, - Err(e) => return Err(e).wrap_with_cost(cost), - }; - appended += 1; - hash_count += u64::from(r.hash_count); - if r.compacted { - compactions += 1; - } - bulk_state_root = r.bulk_state_root; - } - - // Flush MMR nodes staged during compaction so the seeded state is - // fully persisted; callers don't need a separate commit_mmr(). - if let Err(e) = self.commit_mmr() { - // codecov:ignore — commit_mmr only fails on a storage fault during - // the final MMR flush; this method always flushes within the same - // call after appending, so there's no way to leave a pending overlay - // and then fail only the flush via the test mocks. - return Err(e).wrap_with_cost(cost); - } - - if appended == 0 { - // Nothing appended — report the current (unchanged) state root. - bulk_state_root = match self.bulk_tree.compute_current_state_root() { - Ok(r) => r, - Err(e) => { - return Err(CommitmentTreeError::InvalidData(format!( - "state root: {}", - e - ))) - .wrap_with_cost(cost); - } - }; - } - - Ok(BulkSeedSummary { - appended, - total_count: self.bulk_tree.total_count, - bulk_state_root, - hash_count, - compactions, - }) - .wrap_with_cost(cost) - } -} diff --git a/grovedb-commitment-tree/src/commitment_tree/tests.rs b/grovedb-commitment-tree/src/commitment_tree/tests.rs index 6ccb89087..006290fa9 100644 --- a/grovedb-commitment-tree/src/commitment_tree/tests.rs +++ b/grovedb-commitment-tree/src/commitment_tree/tests.rs @@ -20,22 +20,14 @@ mod storage_tests { /// /// Only `get` and `put` are functional — the rest are stubs /// since `CommitmentTree` only uses data storage operations. - /// - /// `fail_get` / `fail_put` are shared toggles used by the fault-injection - /// tests to make storage reads/writes fail *after* construction, exercising - /// the otherwise-unreachable storage-error branches. struct MockDataStorageContext { data: std::cell::RefCell, Vec>>, - fail_get: std::rc::Rc>, - fail_put: std::rc::Rc>, } impl MockDataStorageContext { fn new() -> Self { Self { data: std::cell::RefCell::new(BTreeMap::new()), - fail_get: Default::default(), - fail_put: Default::default(), } } @@ -45,22 +37,8 @@ mod storage_tests { data.insert(key.to_vec(), value); Self { data: std::cell::RefCell::new(data), - fail_get: Default::default(), - fail_put: Default::default(), } } - - /// Clone the (get, put) failure toggles so a test can flip them after the - /// context has been moved into a `CommitmentTree`. - #[cfg(feature = "test-seeding-ct")] - fn fault_handles( - &self, - ) -> ( - std::rc::Rc>, - std::rc::Rc>, - ) { - (self.fail_get.clone(), self.fail_put.clone()) - } } struct StubBatch; @@ -185,15 +163,6 @@ mod storage_tests { _children_sizes: ChildrenSizesWithIsSumTree, _cost_info: Option, ) -> CostResult<(), grovedb_storage::Error> { - if self.fail_put.get() { - return Err(grovedb_storage::Error::StorageError( - "injected put failure".to_string(), - )) - .wrap_with_cost(OperationCost { - seek_count: 1, - ..Default::default() - }); - } self.data .borrow_mut() .insert(key.as_ref().to_vec(), value.to_vec()); @@ -207,15 +176,6 @@ mod storage_tests { &self, key: K, ) -> CostResult>, grovedb_storage::Error> { - if self.fail_get.get() { - return Err(grovedb_storage::Error::StorageError( - "injected get failure".to_string(), - )) - .wrap_with_cost(OperationCost { - seek_count: 1, - ..Default::default() - }); - } let store = self.data.borrow(); let val = store.get(key.as_ref()).cloned(); let loaded = val.as_ref().map_or(0, |v| v.len() as u64); @@ -1008,10 +968,6 @@ mod storage_tests { ); } - // The frontier/bulk consistency check is removed under `test-seeding-ct` - // (frontier-less seeding intentionally desyncs the two), so this test only - // applies when that feature is off. - #[cfg(not(feature = "test-seeding-ct"))] #[test] fn test_open_frontier_total_count_mismatch() { // 1. Create a tree, append 1 item, save @@ -1066,10 +1022,9 @@ mod storage_tests { ); } - // ── Frontier-less seeding (test-seeding-ct feature) ────────────────────── + // ── Batched append (append_many_raw) ──────────────────────────────── /// A correctly-sized DashMemo payload filled with a deterministic pattern. - #[cfg(feature = "test-seeding-ct")] fn seed_payload(index: u8) -> Vec { let mut p = vec![0u8; ciphertext_payload_size::()]; p[0] = index; @@ -1077,277 +1032,239 @@ mod storage_tests { p } - #[cfg(feature = "test-seeding-ct")] - #[test] - fn test_append_raw_without_frontier_does_not_touch_frontier() { - let ctx = MockDataStorageContext::new(); + /// Build a fresh tree, append all `entries` one-by-one via [`append_raw`]. + fn build_via_per_leaf_append( + entries: &[([u8; 32], [u8; 32], Vec)], + ) -> CommitmentTree { let mut ct = - CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); - - let r = ct - .append_raw_without_frontier(test_leaf(0), test_rho(0), &seed_payload(0)) - .value - .expect("frontier-less append should succeed"); - - assert_eq!(r.global_position, 0, "first append is position 0"); - assert_eq!(ct.total_count(), 1, "bulk tree advanced"); - // Frontier untouched: still empty. - assert_eq!(ct.tree_size(), 0, "frontier must remain empty"); - assert_eq!(ct.position(), None, "frontier has no position"); - assert_eq!( - ct.root_hash(), - CommitmentFrontier::new().root_hash(), - "anchor must equal the empty-frontier root" - ); + CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, MockDataStorageContext::new()) + .expect("new should succeed"); + for (cmx, rho, payload) in entries { + ct.append_raw(*cmx, *rho, payload) + .value + .expect("per-leaf append_raw should succeed"); + } + ct } - #[cfg(feature = "test-seeding-ct")] - #[test] - fn test_append_raw_without_frontier_rejects_wrong_payload_size() { - let ctx = MockDataStorageContext::new(); + /// Build a fresh tree by passing all `entries` to [`append_many_raw`] in one call. + fn build_via_append_many_raw( + entries: Vec<([u8; 32], [u8; 32], Vec)>, + ) -> CommitmentTree { let mut ct = - CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); - - let result = ct.append_raw_without_frontier(test_leaf(0), test_rho(0), &[0u8; 7]); - assert!( - result.value.is_err(), - "should reject a payload of the wrong size" - ); - assert_eq!(ct.total_count(), 0, "tree must not be mutated on rejection"); + CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, MockDataStorageContext::new()) + .expect("new should succeed"); + ct.append_many_raw(entries) + .value + .expect("append_many_raw should succeed"); + ct } - #[cfg(feature = "test-seeding-ct")] - #[test] - fn test_append_raw_without_frontier_accepts_non_pallas_cmx() { - let ctx = MockDataStorageContext::new(); - let mut ct = - CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); - - // All-0xFF is NOT a valid Pallas field element; the frontier-less path - // accepts it anyway (no frontier validation). - let result = ct.append_raw_without_frontier([0xFF; 32], test_rho(1), &seed_payload(1)); - assert!( - result.value.is_ok(), - "frontier-less append must accept arbitrary cmx filler" - ); - assert_eq!(ct.total_count(), 1); + /// Build a deterministic test sequence of `n` entries. + fn make_entries(n: u64) -> Vec<([u8; 32], [u8; 32], Vec)> { + (0..n) + .map(|i| { + ( + test_leaf(i), + test_rho((i % 256) as u8), + seed_payload((i % 256) as u8), + ) + }) + .collect() } - #[cfg(feature = "test-seeding-ct")] + /// Byte-for-byte equivalence: for the same input sequence, `append_many_raw` + /// must produce the same `CommitmentFrontier` state and the same + /// BulkAppendTree state root as N × `append_raw`. This is the core + /// invariant — if it ever fails, batched callers can produce anchors that + /// don't match what the per-leaf path would have produced. #[test] - fn test_append_many_without_frontier_seeds_and_reopens() { - let ctx = MockDataStorageContext::new(); - let mut ct = - CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); - - const N: u64 = 12; - let notes = (0..N).map(|i| (test_leaf(i), test_rho(i as u8), seed_payload(i as u8))); - let summary = ct - .append_many_without_frontier(notes) - .value - .expect("bulk seed should succeed"); - - assert_eq!(summary.appended, N); - assert_eq!(summary.total_count, N); - assert_eq!(ct.total_count(), N); - // TEST_CHUNK_POWER=1 → epoch_size 2, so seeding 12 notes finalizes - // several chunks. - assert!( - summary.compactions > 0, - "small epoch size should trigger compactions" - ); - assert!(summary.hash_count > 0, "appends perform blake3 hashes"); - // Frontier left empty by design. - assert_eq!(ct.tree_size(), 0, "frontier must remain empty"); - - // The summary's bulk_state_root matches a fresh computation. - let live_root = ct - .bulk_tree - .compute_current_state_root() - .expect("state root"); - assert_eq!(summary.bulk_state_root, live_root); - - // Re-open the seeded (frontier-less) tree: tolerated under test-seeding-ct. - let storage = ct.bulk_tree.dense_tree.storage; - let loaded = CommitmentTree::<_, DashMemo>::open(N, TEST_CHUNK_POWER, storage) - .value - .expect("open should tolerate empty frontier under test-seeding-ct"); - assert_eq!(loaded.total_count(), N, "reopened total_count matches"); - assert_eq!(loaded.tree_size(), 0, "reopened frontier still empty"); - assert_eq!( - loaded - .bulk_tree - .compute_current_state_root() - .expect("state root"), - live_root, - "bulk state root survives the round-trip" - ); + fn append_many_raw_byte_for_byte_matches_per_leaf() { + // 0 / 1 / 2 / 3 cover edge cases around the very-empty and pre-compaction + // shapes; 100 spans the buffer mid-range; 2048 fills exactly one epoch + // (with TEST_CHUNK_POWER=1 we hit MANY compactions, exercising the cache + // + MMR path); 10_000 spans several epochs at meaningful scale. + for n in [0u64, 1, 2, 3, 100, 2048, 10_000] { + let entries = make_entries(n); + let a = build_via_per_leaf_append(&entries); + let b = build_via_append_many_raw(entries); + + assert_eq!( + a.root_hash(), + b.root_hash(), + "Sinsemilla root mismatch at N={}: per-leaf vs append_many_raw", + n + ); + assert_eq!( + a.frontier.serialize(), + b.frontier.serialize(), + "frontier serialization mismatch at N={}: per-leaf vs append_many_raw", + n + ); + let a_state = a.compute_current_state_root().expect("state root A"); + let b_state = b.compute_current_state_root().expect("state root B"); + assert_eq!( + a_state, b_state, + "bulk tree state_root mismatch at N={}: per-leaf vs append_many_raw", + n + ); + assert_eq!( + a.total_count(), + b.total_count(), + "total_count mismatch at N={}: per-leaf vs append_many_raw", + n + ); + assert_eq!(a.total_count(), n, "tree should hold exactly N={} items", n); + } } - #[cfg(feature = "test-seeding-ct")] + /// `CommitmentFrontier::append_no_root` is just the carry-chain part of the + /// upstream `Frontier::append`. Its [`OperationCost`] must therefore omit + /// the depth-32 root walk that the eager [`append`] would have counted. + /// + /// Pairing N × `append_no_root` with a single `root_hash_with_cost` recovers + /// the depth walk *once*, exactly as the batched API does in production — + /// so this is also the structural check that justifies the speedup. #[test] - fn test_append_raw_without_frontier_surfaces_bulk_storage_error() { - let ctx = MockDataStorageContext::new(); - let (fail_get, fail_put) = ctx.fault_handles(); - let mut ct = - CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); + fn append_no_root_cost_omits_per_leaf_depth_walk() { + // We use a small N here to keep the test fast — the relationship is + // independent of N. + const N: u64 = 16; + + let mut frontier_eager = CommitmentFrontier::new(); + let mut frontier_lazy = CommitmentFrontier::new(); + let mut eager_cost = OperationCost::default(); + let mut lazy_cost = OperationCost::default(); + + for i in 0..N { + let cmx = test_leaf(i); + frontier_eager + .append(cmx) + .unwrap_add_cost(&mut eager_cost) + .expect("eager append"); + frontier_lazy + .append_no_root(cmx) + .unwrap_add_cost(&mut lazy_cost) + .expect("lazy append"); + } - // Make the underlying BulkAppendTree append fail on storage I/O so the - // wrapped "bulk append" error branch is exercised. - fail_get.set(true); - fail_put.set(true); + // Both paths must produce identical frontier state. + assert_eq!(frontier_eager.root_hash(), frontier_lazy.root_hash()); + assert_eq!(frontier_eager.serialize(), frontier_lazy.serialize()); - let err = ct - .append_raw_without_frontier(test_leaf(0), test_rho(0), &seed_payload(0)) - .value - .expect_err("bulk storage failure should surface"); - assert!( - format!("{}", err).contains("bulk append"), - "error should be wrapped as a bulk append failure: {}", - err + // The eager path counts 32 Sinsemilla hashes per call for the depth + // walk; the lazy path counts only the carry chain. Difference is + // exactly 32 × N. + assert_eq!( + eager_cost.sinsemilla_hash_calls - lazy_cost.sinsemilla_hash_calls, + 32 * N as u32, + "eager append must include 32 depth-walk hashes per leaf; lazy must not" ); - } - - #[cfg(feature = "test-seeding-ct")] - #[test] - fn test_append_many_without_frontier_propagates_entry_error() { - let ctx = MockDataStorageContext::new(); - let mut ct = - CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); - // A wrong-size payload makes the per-entry append fail; the bulk loop - // must propagate that error rather than swallow it. - let bad_payload = vec![0u8; 1]; - let err = ct - .append_many_without_frontier(std::iter::once((test_leaf(0), test_rho(0), bad_payload))) - .value - .expect_err("a bad entry must propagate out of the bulk loop"); - assert!( - matches!(err, CommitmentTreeError::InvalidPayloadSize { .. }), - "expected the per-entry payload-size error to propagate, got: {}", - err + // Recover the depth walk *once* via root_hash_with_cost. The lazy + // path then matches the eager path minus the deferred walks + // (32 × (N − 1)). + let root_ctx = frontier_lazy.root_hash_with_cost(); + let lazy_total = lazy_cost.sinsemilla_hash_calls + root_ctx.cost.sinsemilla_hash_calls; + assert_eq!( + eager_cost.sinsemilla_hash_calls, + lazy_total + 32 * (N as u32 - 1), + "N × append == N × append_no_root + 1 × root_hash + 32 × (N − 1)" ); - } - #[cfg(feature = "test-seeding-ct")] - #[test] - fn test_append_many_without_frontier_surfaces_state_root_error() { - let ctx = MockDataStorageContext::new(); - let (fail_get, _fail_put) = ctx.fault_handles(); - let mut ct = - CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); - - // Seed a few notes so the tree has persisted state to read back. - let notes = (0..3u8).map(|i| (test_leaf(i as u64), test_rho(i), seed_payload(i))); - ct.append_many_without_frontier(notes) - .value - .expect("seed should succeed"); - - // An empty follow-up call recomputes the current state root; with reads - // failing, that recomputation must surface the error. - fail_get.set(true); - let res = - ct.append_many_without_frontier(std::iter::empty::<([u8; 32], [u8; 32], Vec)>()); + // Sanity: lazy carry-chain cost is bounded by N (much less than 32 × N). assert!( - res.value.is_err(), - "state-root recomputation read failure should surface" + lazy_cost.sinsemilla_hash_calls < N as u32, + "carry-chain cost should be amortized ~O(1) per leaf, not 32" ); } - #[cfg(feature = "test-seeding-ct")] - #[test] - fn test_add_real_notes_after_frontier_less_seed_then_reopen() { - let ctx = MockDataStorageContext::new(); - let mut ct = - CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); - - // Seed filler frontier-less (frontier stays empty). - const SEEDED: u64 = 6; - let notes = (0..SEEDED).map(|i| (test_leaf(i), test_rho(i as u8), seed_payload(i as u8))); - ct.append_many_without_frontier(notes) - .value - .expect("seed should succeed"); - assert_eq!(ct.tree_size(), 0, "frontier empty after seeding"); - - // Now add a real, frontier-tracked note on top via the normal path. - ct.append(test_leaf(100), test_rho(100), &test_ciphertext(100)) - .value - .expect("normal append on a seeded tree should succeed"); - ct.save().value.expect("save should succeed"); - - let total = ct.total_count(); - assert_eq!(total, SEEDED + 1, "bulk advanced for the real note"); - assert_eq!(ct.tree_size(), 1, "frontier holds only the post-seed note"); - - // Reopen at the mismatched (frontier_size=1, total_count=SEEDED+1): - // tolerated because the consistency check is dropped under the feature. - let storage = ct.bulk_tree.dense_tree.storage; - let loaded = CommitmentTree::<_, DashMemo>::open(total, TEST_CHUNK_POWER, storage) - .value - .expect("reopen of a seeded+appended tree should succeed under test-seeding-ct"); - assert_eq!(loaded.total_count(), total); - assert_eq!(loaded.tree_size(), 1); - } - - #[cfg(feature = "test-seeding-ct")] + /// The Sinsemilla anchor produced by `append_many_raw` must be **spend- + /// usable** — i.e. an Orchard Merkle auth path for a leaf at a known + /// position must verify against the batched-built anchor, exactly as it + /// would against an anchor built leaf-by-leaf. + /// + /// We compute the auth path independently (pairwise Sinsemilla combine to + /// the root, padding upper levels with `empty_root`), then use the orchard + /// `MerklePath` API to derive an anchor from `(position, path, cmx)` and + /// require equality with `ct.anchor()`. #[test] - fn test_frontier_less_append_rejected_on_non_empty_frontier() { - let ctx = MockDataStorageContext::new(); - let mut ct = - CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); + fn append_many_raw_anchor_is_spend_usable() { + use crate::{ExtractedNoteCommitment, MerkleHashOrchard, MerklePath}; + + // Modest N keeps the manual auth-path recomputation cheap; the + // verification property is N-agnostic, so a smaller N proves the + // anchor shape. + const N: u64 = 257; + const TARGET_POSITION: usize = 113; + + let entries = make_entries(N); + let owned_cmx_bytes = entries[TARGET_POSITION].0; + let ct = build_via_append_many_raw(entries.clone()); + let anchor = ct.anchor(); - // Build a non-empty frontier via a normal append first. - ct.append(test_leaf(0), test_rho(0), &test_ciphertext(0)) - .value - .expect("normal append should succeed"); - assert_eq!(ct.tree_size(), 1, "frontier should be non-empty"); - let total_before = ct.total_count(); + // Independent auth-path recomputation over the same leaf set. + let leaves: Vec = entries + .iter() + .map(|(cmx, _, _)| { + Option::from(MerkleHashOrchard::from_bytes(cmx)) + .expect("test_leaf produces valid Pallas elements") + }) + .collect(); - // Both frontier-less entry points must refuse to advance the bulk tree. - let single = ct.append_raw_without_frontier(test_leaf(1), test_rho(1), &seed_payload(1)); - assert!( - single.value.is_err(), - "single frontier-less append must be rejected on a non-empty frontier" - ); + let auth_path = compute_orchard_auth_path(&leaves, TARGET_POSITION); + let target_cmx = Option::::from( + ExtractedNoteCommitment::from_bytes(&owned_cmx_bytes), + ) + .expect("test_leaf produces valid Pallas elements"); - let bulk = ct.append_many_without_frontier(std::iter::once(( - test_leaf(2), - test_rho(2), - seed_payload(2), - ))); - assert!( - bulk.value.is_err(), - "bulk frontier-less seed must be rejected on a non-empty frontier" - ); + let merkle_path = MerklePath::from_parts(TARGET_POSITION as u32, auth_path); + let computed = merkle_path.root(target_cmx); assert_eq!( - ct.total_count(), - total_before, - "rejected frontier-less appends must not mutate the tree" + computed, anchor, + "auth path against batched anchor failed — anchor is not spend-usable" ); } - #[cfg(feature = "test-seeding-ct")] - #[test] - fn test_append_many_without_frontier_empty_input() { - let ctx = MockDataStorageContext::new(); - let mut ct = - CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); - - let summary = ct - .append_many_without_frontier(std::iter::empty()) - .value - .expect("empty seed should succeed"); + /// Build the Orchard Sinsemilla auth path for `position` over `leaves`, + /// padding upper levels with `MerkleHashOrchard::empty_root(level)` once + /// the tree thins beyond the leaf set. + fn compute_orchard_auth_path( + leaves: &[crate::MerkleHashOrchard], + position: usize, + ) -> [crate::MerkleHashOrchard; 32] { + use crate::{Hashable, Level, MerkleHashOrchard}; + + let mut current: Vec = leaves.to_vec(); + let mut path: Vec = Vec::with_capacity(32); + let mut idx = position; + + for level in 0..32u8 { + let l = Level::from(level); + let empty = MerkleHashOrchard::empty_root(l); + + let sibling_idx = idx ^ 1; + let sibling = current.get(sibling_idx).copied().unwrap_or(empty); + path.push(sibling); + + // Combine pairs up to the next level. Pad with `empty` so the + // upper-level positions align with `idx /= 2` regardless of how + // many leaves there are. + let next_len = current.len().div_ceil(2); + let mut next: Vec = Vec::with_capacity(next_len); + let mut i = 0; + while i < current.len() { + let left = current[i]; + let right = current.get(i + 1).copied().unwrap_or(empty); + next.push(MerkleHashOrchard::combine(l, &left, &right)); + i += 2; + } + current = next; + idx /= 2; + } - assert_eq!(summary.appended, 0); - assert_eq!(summary.total_count, 0); - assert_eq!(summary.compactions, 0); - assert_eq!( - summary.bulk_state_root, - ct.bulk_tree - .compute_current_state_root() - .expect("state root"), - "empty seed reports the current state root" - ); + path.try_into() + .expect("32 levels of auth path produced from the loop above") } } diff --git a/grovedb-commitment-tree/src/lib.rs b/grovedb-commitment-tree/src/lib.rs index e09c17a1a..7257a4ff9 100644 --- a/grovedb-commitment-tree/src/lib.rs +++ b/grovedb-commitment-tree/src/lib.rs @@ -67,8 +67,6 @@ pub use commitment_tree::{ ciphertext_payload_size, deserialize_ciphertext, serialize_ciphertext, CommitmentAppendResult, CommitmentTree, COMMITMENT_TREE_DATA_KEY, }; -#[cfg(feature = "test-seeding-ct")] -pub use commitment_tree::{BulkSeedSummary, FrontierLessAppendResult}; pub use error::CommitmentTreeError; #[cfg(feature = "server")] pub use grovedb_bulk_append_tree::{ diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index e9e9b2489..3e3d069b2 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -108,11 +108,6 @@ verify = [ ] estimated_costs = ["full"] zk_client = ["grovedb-commitment-tree", "grovedb-commitment-tree/client"] -# TEST / DEVNET ONLY — forwards `grovedb-commitment-tree/test-seeding-ct`. -# Exposes the frontier-less commitment-tree seeding methods and relaxes the -# frontier/bulk consistency check in `CommitmentTree::open` so a node can read -# seeded (anchor-less) devnet state. Never enable in production. -test-seeding-ct = ["minimal", "grovedb-commitment-tree/test-seeding-ct"] grovedbg = [ "grovedbg-types", "tokio", From bdc0d6e9d7a5f7cec47b2701d08d6e35f558c7b8 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 28 May 2026 01:40:38 +0200 Subject: [PATCH 10/16] docs(commitment-tree): add --features server to seeding bench example commands The bench is gated by required-features = ["server"], so the example invocations in the file-level docs need --features server or they run the fallback `main` and skip the actual benchmark. Addresses CodeRabbit review on #751. Co-Authored-By: Claude Opus 4.7 (1M context) --- grovedb-commitment-tree/benches/seeding.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grovedb-commitment-tree/benches/seeding.rs b/grovedb-commitment-tree/benches/seeding.rs index b90542760..5e072ef06 100644 --- a/grovedb-commitment-tree/benches/seeding.rs +++ b/grovedb-commitment-tree/benches/seeding.rs @@ -9,12 +9,12 @@ //! //! Run with: //! ```text -//! cargo bench -p grovedb-commitment-tree --bench seeding +//! cargo bench -p grovedb-commitment-tree --bench seeding --features server //! ``` //! //! The note count defaults to 1,000,000 and can be overridden: //! ```text -//! SEED_N=200000 cargo bench -p grovedb-commitment-tree --bench seeding +//! SEED_N=200000 cargo bench -p grovedb-commitment-tree --bench seeding --features server //! ``` #[cfg(feature = "server")] From 5eb7a5380a6e974513343352acfd6b30a8c1f87c Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 28 May 2026 12:14:46 +0200 Subject: [PATCH 11/16] fix(commitment-tree): don't commit_mmr inside append_many_raw MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GroveDB's `StorageContext::get` reads straight from the transaction and never consults its `StorageBatch` (see `storage/src/rocksdb_storage/storage_context/ context_tx.rs:296-310`). So as soon as `commit_mmr` drains the MMR overlay into the batch, the freshly-flushed peaks become invisible to reads until `commit_multi_context_batch` lands the batch in the tx — which only happens at the very end of a session. For chained `append_many_raw` calls on the same `CommitmentTree` (the 500k-note Drive seeder pattern in dashpay/platform#3732), the internal `commit_mmr` at the end of batch N drained the overlay; batch N+1's first compaction then read a peak via `MmrStore::element_at_position` → `ctx.get` → tx (peak not there) → `MMR::push` raised `InconsistentStore`. Make `commit_mmr` the caller's responsibility — same as `append_raw`, which also never flushes on its own. The overlay now stays alive across chained batches; the caller flushes it once, right before committing the surrounding batch. The docs spell this out and explicitly warn against mid-session `commit_mmr`. Updated the seeding bench to call `commit_mmr` explicitly before dropping `ct`. The byte-for-byte equivalence, spend-usable anchor, and cost-accounting tests are unaffected (they compare in-memory frontier/state-root values that don't depend on commit ordering). Co-Authored-By: Claude Opus 4.7 (1M context) --- grovedb-commitment-tree/benches/seeding.rs | 5 +++ .../src/commitment_tree/mod.rs | 35 ++++++++++++++----- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/grovedb-commitment-tree/benches/seeding.rs b/grovedb-commitment-tree/benches/seeding.rs index 5e072ef06..a24458c57 100644 --- a/grovedb-commitment-tree/benches/seeding.rs +++ b/grovedb-commitment-tree/benches/seeding.rs @@ -83,6 +83,11 @@ fn main() { .append_many_raw(notes) .value .expect("batched commitment-tree seeding"); + // Flush the MMR overlay into the storage batch now that we're done + // appending. Must come before dropping `ct` (which would lose the overlay) + // and before committing the storage batch (which writes to disk). + ct.commit_mmr().expect("commit_mmr"); + let seed_elapsed = t_seed.elapsed(); // Release the storage context's borrow of the batch before committing. diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs index 2208bbaa1..373f9fbe4 100644 --- a/grovedb-commitment-tree/src/commitment_tree/mod.rs +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -358,7 +358,25 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { /// matches the per-leaf behavior of calling [`append_raw`](Self::append_raw) /// in a loop. /// - /// Call [`save`](Self::save) afterwards to persist the updated frontier. + /// # Persistence — caller responsibilities + /// + /// Like [`append_raw`](Self::append_raw), this method does **not** flush + /// state to disk on its own. After your final batch — i.e. just before + /// committing the surrounding `StorageBatch` / transaction — the caller + /// **must** call: + /// + /// 1. [`commit_mmr`](Self::commit_mmr) to write MMR nodes staged in the + /// overlay during compactions, and + /// 2. [`save`](Self::save) to persist the Sinsemilla frontier. + /// + /// **Do not call [`commit_mmr`](Self::commit_mmr) between chained + /// `append_many_raw` calls.** A GroveDB `StorageContext::get` does not see + /// writes that are sitting in its `StorageBatch` (reads go straight to the + /// underlying transaction). Flushing the overlay mid-session would put the + /// MMR peaks into the batch, where the *next* batch's compactions can't + /// read them, and the second batch would then fail with `InconsistentStore`. + /// Keeping the overlay alive across chained calls is what lets later + /// compactions resolve their sibling reads in memory. pub fn append_many_raw( &mut self, entries: I, @@ -454,14 +472,13 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { cost += root_ctx.cost; let sinsemilla_root = root_ctx.value; - // Flush MMR nodes staged during compaction so the seeded state is fully - // persisted; callers don't need a separate `commit_mmr` after this. - if let Err(e) = self.commit_mmr() { - // codecov:ignore — only reachable on a storage fault during the - // final MMR flush; the bench / seed path can't isolate that fault - // from the appends with our test mocks. - return Err(e).wrap_with_cost(cost); - } + // Intentionally do NOT call `commit_mmr` here. Any nodes staged in the + // MMR overlay during compactions must stay in memory until the caller + // finishes the whole session — see the "Persistence" section in the + // method docs. Flushing here would put the peaks into the surrounding + // `StorageBatch`, where the next chained `append_many_raw` would be + // unable to read them (GroveDB `get` does not see batched writes), and + // the next compaction would fail with `InconsistentStore`. Ok(CommitmentAppendResult { sinsemilla_root, From bd36a1d29c195aaf85f62dae0304f3518a13ab01 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Mon, 1 Jun 2026 17:45:16 +0700 Subject: [PATCH 12/16] style: cargo fmt --all Three minor wrap/line-length fixes flagged by CI after the develop merge: - grovedb-bulk-append-tree/src/tree/append.rs: rejoin one-line import - grovedb/src/operations/commitment_tree.rs: collapse one-line let - storage/src/rocksdb_storage/storage.rs: collapse multi-line fn signature Co-Authored-By: Claude Opus 4.7 --- grovedb-bulk-append-tree/src/tree/append.rs | 3 +-- grovedb/src/operations/commitment_tree.rs | 3 +-- storage/src/rocksdb_storage/storage.rs | 6 +----- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/grovedb-bulk-append-tree/src/tree/append.rs b/grovedb-bulk-append-tree/src/tree/append.rs index 0e262efd7..9ad0fc54e 100644 --- a/grovedb-bulk-append-tree/src/tree/append.rs +++ b/grovedb-bulk-append-tree/src/tree/append.rs @@ -7,8 +7,7 @@ use grovedb_storage::StorageContext; use super::{ capacity_for_height, hash::compute_state_root, AppendManyResult, AppendNoStateRootResult, - AppendResult, - BulkAppendTree, + AppendResult, BulkAppendTree, }; use crate::{chunk::serialize_chunk_blob, BulkAppendError}; diff --git a/grovedb/src/operations/commitment_tree.rs b/grovedb/src/operations/commitment_tree.rs index 17d521bfa..47f33aba3 100644 --- a/grovedb/src/operations/commitment_tree.rs +++ b/grovedb/src/operations/commitment_tree.rs @@ -295,8 +295,7 @@ impl GroveDb { ) ); - let updated_element = - Element::new_commitment_tree(new_total_count, chunk_power, flags); + let updated_element = Element::new_commitment_tree(new_total_count, chunk_power, flags); cost_return_on_error_into!( &mut cost, diff --git a/storage/src/rocksdb_storage/storage.rs b/storage/src/rocksdb_storage/storage.rs index 9a1493e47..612352773 100644 --- a/storage/src/rocksdb_storage/storage.rs +++ b/storage/src/rocksdb_storage/storage.rs @@ -513,11 +513,7 @@ impl RocksDbStorage { /// /// The ingest happens at the DB level and bypasses any open transaction. /// Callers must arrange for txn semantics at a higher layer. - pub fn ingest_subtree_sst( - &self, - cf_name: &str, - sst_path: &Path, - ) -> Result<(), Error> { + pub fn ingest_subtree_sst(&self, cf_name: &str, sst_path: &Path) -> Result<(), Error> { let cf_handle = self .db .cf_handle(cf_name) From 67acdbcb610d8ff34c8ebeffd8262fa2475e9b32 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Mon, 1 Jun 2026 18:10:32 +0700 Subject: [PATCH 13/16] chore: drop vestigial BulkAppendTree::append_many helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #751 ('perf(commitment-tree): batched append_many_raw — replace _without_frontier API') chose to inline the bulk-append loop inside CommitmentTree::append_many_raw rather than extract a BulkAppendTree-level batched helper. This branch carried our pre-#751 helper version (defining BulkAppendTree::append_many and AppendManyResult), which has no callers on develop or in platform — only append_many_raw is used. Reset both files to develop's versions so the branch's net delta vs develop is only the snapshot bootstrap surface (ingest_subtree_sst, replace_commitment_tree_subtree_root, raw_storage). Co-Authored-By: Claude Opus 4.7 --- grovedb-bulk-append-tree/src/tree/append.rs | 51 +-------------------- grovedb-bulk-append-tree/src/tree/mod.rs | 19 -------- 2 files changed, 2 insertions(+), 68 deletions(-) diff --git a/grovedb-bulk-append-tree/src/tree/append.rs b/grovedb-bulk-append-tree/src/tree/append.rs index 9ad0fc54e..a3ca53b5c 100644 --- a/grovedb-bulk-append-tree/src/tree/append.rs +++ b/grovedb-bulk-append-tree/src/tree/append.rs @@ -6,8 +6,8 @@ use grovedb_merkle_mountain_range::{ use grovedb_storage::StorageContext; use super::{ - capacity_for_height, hash::compute_state_root, AppendManyResult, AppendNoStateRootResult, - AppendResult, BulkAppendTree, + capacity_for_height, hash::compute_state_root, AppendNoStateRootResult, AppendResult, + BulkAppendTree, }; use crate::{chunk::serialize_chunk_blob, BulkAppendError}; @@ -128,53 +128,6 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { }) } - /// Batch-append a sequence of values, computing the state root **once** at - /// the end instead of once per leaf. - /// - /// Byte-for-byte equivalent to calling [`append`](Self::append) per value - /// (same dense-tree state, same MMR, same final `state_root`), but skips - /// the per-leaf blake3 state-root computation. Used by - /// [`CommitmentTree::append_many_raw`] to keep large bulk seeds linear. - /// - /// On error the tree is left partially mutated — discard the surrounding - /// transaction if you need all-or-nothing semantics. - /// - /// [`CommitmentTree::append_many_raw`]: ../../grovedb_commitment_tree/struct.CommitmentTree.html#method.append_many_raw - pub fn append_many(&mut self, values: I) -> Result - where - I: IntoIterator>, - { - let mut appended: u64 = 0; - let mut hash_count: u64 = 0; - let mut compactions: u64 = 0; - let mut last_global_position: Option = None; - - for value in values { - let r = self.append_no_state_root(&value)?; - last_global_position = Some(r.global_position); - hash_count = hash_count.saturating_add(u64::from(r.hash_count)); - if r.compacted { - compactions += 1; - } - appended += 1; - } - - // One blake3 for the final state root (matching the +1 each per-leaf - // `append` would have counted). - let state_root = self.compute_current_state_root()?; - if appended > 0 { - hash_count = hash_count.saturating_add(1); - } - - Ok(AppendManyResult { - state_root, - last_global_position, - appended, - hash_count, - compactions, - }) - } - /// Compute the current state root without modifying the tree. /// /// Uses the cached MMR root when available, so this is O(1) on the diff --git a/grovedb-bulk-append-tree/src/tree/mod.rs b/grovedb-bulk-append-tree/src/tree/mod.rs index 16e7a0f3c..bcea9a521 100644 --- a/grovedb-bulk-append-tree/src/tree/mod.rs +++ b/grovedb-bulk-append-tree/src/tree/mod.rs @@ -57,25 +57,6 @@ pub struct AppendNoStateRootResult { pub compacted: bool, } -/// Result returned by [`BulkAppendTree::append_many`]. -#[cfg(feature = "storage")] -#[derive(Debug, Clone)] -pub struct AppendManyResult { - /// State root after all appends. Equal to what the final per-leaf - /// [`AppendResult::state_root`] would have been. - pub state_root: [u8; 32], - /// 0-based global position of the last appended value, or `None` if the - /// input iterator was empty. - pub last_global_position: Option, - /// Number of values actually appended (the iterator's length). - pub appended: u64, - /// Sum of blake3 hashes across the batch, including the single end-of-batch - /// state-root computation. - pub hash_count: u64, - /// Number of compactions (epoch flushes) that occurred during the batch. - pub compactions: u64, -} - /// Compute MMR size from leaf count: `2 * n - popcount(n)`. /// /// This is a well-known MMR property: the total number of nodes (leaves + From 2e9dff50455083dc9fbf15b954cea602431acf9d Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Mon, 1 Jun 2026 22:04:41 +0700 Subject: [PATCH 14/16] feat: gate snapshot-bootstrap surface behind `unsafe-dump-load` feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three escape-hatch APIs added for caller-driven subtree dump/restore are now Cargo-feature-gated so production builds don't carry any of them: * `GroveDb::raw_storage()` — borrow underlying RocksDbStorage * `RocksDbStorage::ingest_subtree_sst()` — bulk-load SST into a CF * `GroveDb::replace_subtree_root()` — caller-provided child hash Also generalizes the third API. The previous `replace_commitment_tree_subtree_root` took CommitmentTree-specific arguments (total_count, chunk_power, flags) and constructed the Element internally. The same Merk-tail plumbing is identical across all non-Merk tree types (commitment / mmr / bulk-append / dense), so the new shape just takes an arbitrary `Element` plus the caller-computed combined root: pub fn replace_subtree_root<'b, B, P>( &self, path: P, key: &[u8], new_element: Element, new_combined_root: [u8; 32], transaction: TransactionArg, grove_version: &GroveVersion, ) -> CostResult<(), Error> The body cheap-checks that the Element is a tree variant (rejecting Item/Reference, which have no child-hash slot) but leaves hash-vs-state correctness to the caller — same contract as before, just no longer commitment-tree-specific. Co-Authored-By: Claude Opus 4.7 --- grovedb/Cargo.toml | 11 ++ grovedb/src/lib.rs | 7 + grovedb/src/operations/commitment_tree.rs | 96 +------------- grovedb/src/operations/mod.rs | 6 + .../src/operations/replace_subtree_root.rs | 120 ++++++++++++++++++ storage/Cargo.toml | 4 + storage/src/rocksdb_storage/storage.rs | 11 +- 7 files changed, 158 insertions(+), 97 deletions(-) create mode 100644 grovedb/src/operations/replace_subtree_root.rs diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index 3e3d069b2..8f88ef91d 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -119,6 +119,17 @@ grovedbg = [ "zip-extensions", "tempfile", ] +# Snapshot-bootstrap primitives — escape hatches that bypass grovedb's normal +# validation: +# - `GroveDb::raw_storage` (escape hatch to StorageContext) +# - `RocksDbStorage::ingest_subtree_sst` (bulk-load SST into a CF) +# - `GroveDb::replace_subtree_root` (caller-provided child hash, no +# cross-check against actual subtree state) +# Enable ONLY for build-time tooling that needs to dump+restore a subtree +# out-of-band (e.g. devnet shielded-pool snapshot bake/apply). Misuse will +# produce an inconsistent Merk tree — there is no runtime check that the +# caller-provided hash matches the underlying state. +unsafe-dump-load = ["grovedb-storage/unsafe-dump-load"] [build-dependencies] hex-literal = "1.1.0" diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 727256fe9..485a9cb33 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -355,6 +355,10 @@ impl GroveDb { /// /// Stability: this is an escape hatch. The exact `RocksDbStorage` shape /// is subject to change as grovedb's internals evolve. + /// + /// Gated behind the `unsafe-dump-load` feature — production builds should + /// leave it off so this escape hatch isn't even compiled in. + #[cfg(feature = "unsafe-dump-load")] pub fn raw_storage(&self) -> &grovedb_storage::rocksdb_storage::RocksDbStorage { &self.db } @@ -375,6 +379,9 @@ impl GroveDb { /// transaction semantics at a higher layer (e.g. only call when the /// destination subtree is known empty, and rely on InitChain /// abort = wipe-and-restart for failure recovery). + /// + /// Gated behind the `unsafe-dump-load` feature. + #[cfg(feature = "unsafe-dump-load")] pub fn ingest_subtree_sst( &self, cf_name: &str, diff --git a/grovedb/src/operations/commitment_tree.rs b/grovedb/src/operations/commitment_tree.rs index 47f33aba3..8fc5d7021 100644 --- a/grovedb/src/operations/commitment_tree.rs +++ b/grovedb/src/operations/commitment_tree.rs @@ -30,7 +30,7 @@ use grovedb_version::version::GroveVersion; use crate::{ batch::{GroveOp, QualifiedGroveDbOp}, util::TxRef, - Element, ElementFlags, Error, GroveDb, Transaction, TransactionArg, + Element, Error, GroveDb, Transaction, TransactionArg, }; // ── Helpers ────────────────────────────────────────────────────────────── @@ -238,100 +238,6 @@ impl GroveDb { .wrap_with_cost(cost) } - /// Replace the child hash of a CommitmentTree subtree leaf in its parent - /// Merk with a caller-provided `new_combined_root`, simultaneously - /// updating the leaf's `total_count` and `flags`. - /// - /// This is the parent-Merk tail of [`Self::commitment_tree_insert`] - /// extracted as a public method, intended for **snapshot-based bootstrap** - /// only (e.g. the shielded-pool genesis snapshot at devnet `InitChain` - /// time). Normal append flow MUST go through `commitment_tree_insert`. - /// - /// The caller is responsible for ensuring `new_combined_root` actually - /// matches the subtree's underlying state. The intended caller pattern: - /// 1. Ingest the subtree's storage state into the underlying RocksDB - /// (typically via [`grovedb_storage::rocksdb_storage::RocksDbStorage::ingest_subtree_sst`]). - /// 2. Open a `StorageContext` at the subtree path and reconstruct - /// `CommitmentTree` from it. - /// 3. Compute `combined_root` via - /// `grovedb_commitment_tree::compute_commitment_tree_state_root`, - /// binding the Sinsemilla anchor to the bulk-state root. - /// 4. Call this method with the verified `combined_root`. - /// - /// Mismatches between `new_combined_root` and what re-reading the subtree - /// would compute will produce an inconsistent Merk tree (parent's - /// recorded child hash diverges from actual subtree state). This method - /// does NOT detect such mismatches. - /// - /// `chunk_power` must match the value the BulkAppendTree state was built - /// with — same caveat: not validated here. - pub fn replace_commitment_tree_subtree_root<'b, B, P>( - &self, - path: P, - key: &[u8], - new_total_count: u64, - chunk_power: u8, - flags: Option, - new_combined_root: [u8; 32], - transaction: TransactionArg, - grove_version: &GroveVersion, - ) -> CostResult<(), Error> - where - B: AsRef<[u8]> + 'b, - P: Into>, - { - let path: SubtreePath = path.into(); - let mut cost = OperationCost::default(); - let tx = TxRef::new(&self.db, transaction); - - let batch = StorageBatch::new(); - let mut parent_merk = cost_return_on_error!( - &mut cost, - self.open_transactional_merk_at_path( - path.clone(), - tx.as_ref(), - Some(&batch), - grove_version, - ) - ); - - let updated_element = Element::new_commitment_tree(new_total_count, chunk_power, flags); - - cost_return_on_error_into!( - &mut cost, - updated_element.insert_subtree( - &mut parent_merk, - key, - new_combined_root, - None, - grove_version, - ) - ); - - let mut merk_cache = HashMap::new(); - merk_cache.insert(path.clone(), parent_merk); - - cost_return_on_error!( - &mut cost, - self.propagate_changes_with_transaction( - merk_cache, - path, - tx.as_ref(), - &batch, - grove_version, - ) - ); - - cost_return_on_error!( - &mut cost, - self.db - .commit_multi_context_batch(batch, Some(tx.as_ref())) - .map_err(Into::into) - ); - - tx.commit_local().wrap_with_cost(cost) - } - /// Get the Orchard `Anchor` for a CommitmentTree subtree. /// /// Returns the anchor directly as an `orchard::tree::Anchor`, suitable for diff --git a/grovedb/src/operations/mod.rs b/grovedb/src/operations/mod.rs index 2f22aded6..d9e383c03 100644 --- a/grovedb/src/operations/mod.rs +++ b/grovedb/src/operations/mod.rs @@ -26,5 +26,11 @@ pub mod bulk_append_tree; #[cfg(feature = "minimal")] pub mod dense_tree; +/// Caller-driven subtree-root replacement. Bypasses grovedb's normal +/// "compute child hash from subtree state" invariant — see the module-level +/// docs in `replace_subtree_root.rs` for the safety contract. +#[cfg(all(feature = "minimal", feature = "unsafe-dump-load"))] +pub mod replace_subtree_root; + #[cfg(feature = "minimal")] pub use get::{QueryItemOrSumReturnType, MAX_REFERENCE_HOPS}; diff --git a/grovedb/src/operations/replace_subtree_root.rs b/grovedb/src/operations/replace_subtree_root.rs new file mode 100644 index 000000000..3395f71b3 --- /dev/null +++ b/grovedb/src/operations/replace_subtree_root.rs @@ -0,0 +1,120 @@ +//! Caller-driven subtree-root replacement. +//! +//! Replaces a subtree leaf's child hash + element data inside its parent +//! Merk with caller-provided values, **without** re-reading the subtree to +//! compute the hash. This is the tail of every `*_insert` operation +//! (commitment_tree, mmr_tree, bulk_append_tree, dense_tree) factored out as +//! a single generic helper, intended for **snapshot-based bootstrap** only. +//! +//! # Safety contract +//! +//! The caller is responsible for ensuring `new_combined_root` actually +//! matches the subtree's underlying storage state. Intended pattern: +//! +//! 1. Ingest the subtree's raw storage out-of-band (typically via +//! [`grovedb_storage::rocksdb_storage::RocksDbStorage::ingest_subtree_sst`]). +//! 2. Open a `StorageContext` at the subtree path and reconstruct the +//! appropriate typed tree (CommitmentTree / Mmr / BulkAppend / Dense). +//! 3. Compute the post-ingest combined root via that tree's own root fn. +//! 4. Call [`GroveDb::replace_subtree_root`] with the verified hash and a +//! correctly-constructed [`Element`] (e.g. `Element::new_commitment_tree`). +//! +//! Mismatches between `new_combined_root` and what re-reading the subtree +//! would compute produce an inconsistent Merk tree (the parent's recorded +//! child hash diverges from actual subtree state). This method does NOT +//! detect such mismatches at runtime; misuse is silent corruption. +//! +//! Gated behind the `unsafe-dump-load` Cargo feature. + +use std::collections::HashMap; + +use grovedb_costs::{ + cost_return_on_error, cost_return_on_error_into, CostResult, CostsExt, OperationCost, +}; +use grovedb_merk::element::{ + insert::ElementInsertToStorageExtensions, tree_type::ElementTreeTypeExtensions, +}; +use grovedb_path::SubtreePath; +use grovedb_storage::{Storage, StorageBatch}; +use grovedb_version::version::GroveVersion; + +use crate::{util::TxRef, Element, Error, GroveDb, TransactionArg}; + +impl GroveDb { + /// Replace the child hash + element data of a subtree leaf in its parent + /// Merk with caller-provided values. See the + /// [module-level docs](self) for the safety contract. + pub fn replace_subtree_root<'b, B, P>( + &self, + path: P, + key: &[u8], + new_element: Element, + new_combined_root: [u8; 32], + transaction: TransactionArg, + grove_version: &GroveVersion, + ) -> CostResult<(), Error> + where + B: AsRef<[u8]> + 'b, + P: Into>, + { + let mut cost = OperationCost::default(); + + // Cheap sanity check: must be a tree-shaped Element (Item / Reference + // have no child hash slot). Caller still owns hash-vs-state + // correctness. + if new_element.tree_type().is_none() { + return Err(Error::InvalidInput( + "replace_subtree_root: element is not a tree variant", + )) + .wrap_with_cost(cost); + } + + let path: SubtreePath = path.into(); + let tx = TxRef::new(&self.db, transaction); + let batch = StorageBatch::new(); + + let mut parent_merk = cost_return_on_error!( + &mut cost, + self.open_transactional_merk_at_path( + path.clone(), + tx.as_ref(), + Some(&batch), + grove_version, + ) + ); + + cost_return_on_error_into!( + &mut cost, + new_element.insert_subtree( + &mut parent_merk, + key, + new_combined_root, + None, + grove_version, + ) + ); + + let mut merk_cache = HashMap::new(); + merk_cache.insert(path.clone(), parent_merk); + + cost_return_on_error!( + &mut cost, + self.propagate_changes_with_transaction( + merk_cache, + path, + tx.as_ref(), + &batch, + grove_version, + ) + ); + + cost_return_on_error!( + &mut cost, + self.db + .commit_multi_context_batch(batch, Some(tx.as_ref())) + .map_err(Into::into) + ); + + tx.commit_local().wrap_with_cost(cost) + } +} diff --git a/storage/Cargo.toml b/storage/Cargo.toml index f76532bb2..89e3906c5 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -26,3 +26,7 @@ thiserror = { workspace = true } [features] rocksdb_storage = ["rocksdb", "num_cpus", "lazy_static", "tempfile", "blake3", "integer-encoding"] +# See `grovedb`'s `unsafe-dump-load` feature for context. On this crate it +# gates `RocksDbStorage::ingest_subtree_sst` — the RocksDB IngestExternalFile +# wrapper used to bulk-load a precomputed subtree SST. +unsafe-dump-load = ["rocksdb_storage"] diff --git a/storage/src/rocksdb_storage/storage.rs b/storage/src/rocksdb_storage/storage.rs index 612352773..fef86ad3d 100644 --- a/storage/src/rocksdb_storage/storage.rs +++ b/storage/src/rocksdb_storage/storage.rs @@ -40,9 +40,11 @@ use grovedb_path::SubtreePath; use integer_encoding::VarInt; use lazy_static::lazy_static; use rocksdb::{ - checkpoint::Checkpoint, ColumnFamily, ColumnFamilyDescriptor, IngestExternalFileOptions, - OptimisticTransactionDB, Transaction, WriteBatchWithTransaction, DEFAULT_COLUMN_FAMILY_NAME, + checkpoint::Checkpoint, ColumnFamily, ColumnFamilyDescriptor, OptimisticTransactionDB, + Transaction, WriteBatchWithTransaction, DEFAULT_COLUMN_FAMILY_NAME, }; +#[cfg(feature = "unsafe-dump-load")] +use rocksdb::IngestExternalFileOptions; use super::{PrefixedRocksDbImmediateStorageContext, PrefixedRocksDbTransactionContext}; use crate::{ @@ -513,6 +515,11 @@ impl RocksDbStorage { /// /// The ingest happens at the DB level and bypasses any open transaction. /// Callers must arrange for txn semantics at a higher layer. + /// + /// Gated behind the `unsafe-dump-load` feature — production builds (which + /// have no need to bulk-load precomputed subtree state) should leave it + /// off so this API isn't even compiled in. + #[cfg(feature = "unsafe-dump-load")] pub fn ingest_subtree_sst(&self, cf_name: &str, sst_path: &Path) -> Result<(), Error> { let cf_handle = self .db From c7f6dc319e982dff08a1c9fffebf257f8d99d60b Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Tue, 2 Jun 2026 12:34:00 +0700 Subject: [PATCH 15/16] style: cargo fmt (cfg-gated import order) `#[cfg(feature = "unsafe-dump-load")] use rocksdb::IngestExternalFileOptions` should precede the unconditional `use rocksdb::{...}` block per rustfmt's sort rules. Co-Authored-By: Claude Opus 4.7 --- storage/src/rocksdb_storage/storage.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/src/rocksdb_storage/storage.rs b/storage/src/rocksdb_storage/storage.rs index fef86ad3d..96edafbb1 100644 --- a/storage/src/rocksdb_storage/storage.rs +++ b/storage/src/rocksdb_storage/storage.rs @@ -39,12 +39,12 @@ use grovedb_costs::{ use grovedb_path::SubtreePath; use integer_encoding::VarInt; use lazy_static::lazy_static; +#[cfg(feature = "unsafe-dump-load")] +use rocksdb::IngestExternalFileOptions; use rocksdb::{ checkpoint::Checkpoint, ColumnFamily, ColumnFamilyDescriptor, OptimisticTransactionDB, Transaction, WriteBatchWithTransaction, DEFAULT_COLUMN_FAMILY_NAME, }; -#[cfg(feature = "unsafe-dump-load")] -use rocksdb::IngestExternalFileOptions; use super::{PrefixedRocksDbImmediateStorageContext, PrefixedRocksDbTransactionContext}; use crate::{ From 62e1296ac8c2b11c01b9887db17a59f27e31f253 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Tue, 2 Jun 2026 14:33:04 +0700 Subject: [PATCH 16/16] test(grovedb): unsafe-dump-load subtree roundtrip + non-tree rejection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two new tests covering the `unsafe-dump-load`-gated surface in-tree (codecov patch coverage was 0% on the 80 new lines; CodeRabbit + codecov flagged it on PR #752). `unsafe_dump_load_subtree_roundtrip_preserves_root_hash`: Builds a CommitmentTree subtree on GroveDb A (insert 5 leaves), dumps the subtree's storage to an SST file (via `raw_storage` + `SstFileWriter` — the same path consumers will use), then on a fresh GroveDb B with only an empty CommitmentTree skeleton: ingests the SST via `ingest_subtree_sst` and patches the parent Merk leaf via `replace_subtree_root` with the combined_root recomputed from A. The two GroveDb root_hashes must match byte-for-byte. This pins the snapshot-bootstrap contract documented in `operations/replace_subtree_root.rs`: SST-ingest + caller-provided child hash is equivalent to a normal insert path, *provided* the caller's hash matches the underlying state. `replace_subtree_root_rejects_non_tree_element`: Pins the cheap guard in `replace_subtree_root`: passing an Item / Reference element (with no child-hash slot) must surface as `Error::InvalidInput` rather than silently corrupting the parent Merk. Adds `rocksdb` as a dev-dependency of `grovedb` so the SST-write side of the roundtrip can use the same QuantumExplorer rocksdb revision that `grovedb-storage` optionally depends on. Co-Authored-By: Claude Opus 4.7 --- grovedb/Cargo.toml | 4 + grovedb/src/tests/commitment_tree_tests.rs | 185 +++++++++++++++++++++ 2 files changed, 189 insertions(+) diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index 8f88ef91d..f21c94eae 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -59,6 +59,10 @@ rand = { workspace = true } rand_distr = "0.6" assert_matches = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "time", "sync"] } +# For the `unsafe-dump-load` dump+ingest roundtrip test: writing the SST and +# ingesting it both need direct rocksdb types. Pinned to the same revision +# used by `grovedb-storage`'s optional dep. +rocksdb = { git = "https://github.com/QuantumExplorer/rust-rocksdb.git", rev = "52772eea7bcd214d1d07d80aa538b1d24e5015b7" } [[bench]] name = "insertion_benchmark" diff --git a/grovedb/src/tests/commitment_tree_tests.rs b/grovedb/src/tests/commitment_tree_tests.rs index 06b86bf5c..ab984898c 100644 --- a/grovedb/src/tests/commitment_tree_tests.rs +++ b/grovedb/src/tests/commitment_tree_tests.rs @@ -20,6 +20,17 @@ use crate::{ Element, Error, GroveDb, PathQuery, SizedQuery, }; +#[cfg(feature = "unsafe-dump-load")] +use grovedb_commitment_tree::CommitmentTree; +#[cfg(feature = "unsafe-dump-load")] +use grovedb_path::SubtreePath; +#[cfg(feature = "unsafe-dump-load")] +use grovedb_storage::{rocksdb_storage::RocksDbStorage, RawIterator, Storage, StorageContext}; +#[cfg(feature = "unsafe-dump-load")] +use rocksdb::{Options, SstFileWriter}; +#[cfg(feature = "unsafe-dump-load")] +use tempfile::TempDir; + /// Default chunk power for tests (2^10 = 1024, large enough that compaction /// doesn't happen in most tests with only a few items). const TEST_CHUNK_POWER: u8 = 10; @@ -2496,3 +2507,177 @@ fn test_commitment_compaction_transaction_rollback() { "anchor should revert to empty tree after compaction + rollback" ); } + +// =========================================================================== +// unsafe-dump-load: dump-then-restore subtree round-trip +// =========================================================================== +// +// Exercises the three escape-hatch APIs gated behind `unsafe-dump-load` +// (`GroveDb::raw_storage`, `RocksDbStorage::ingest_subtree_sst`, +// `GroveDb::replace_subtree_root`) in a single end-to-end round-trip on a +// real CommitmentTree subtree. The post-restore GroveDb root_hash must equal +// the source's, byte-for-byte, demonstrating the snapshot-bootstrap contract +// described in `replace_subtree_root.rs`'s module docs. + +#[cfg(feature = "unsafe-dump-load")] +#[test] +fn unsafe_dump_load_subtree_roundtrip_preserves_root_hash() { + let grove_version = GroveVersion::latest(); + const N: u8 = 5; + + // --- A: build a commitment-tree subtree, insert N items --- + let db_a = make_empty_grovedb(); + db_a.insert( + EMPTY_PATH, + b"pool", + Element::empty_commitment_tree(TEST_CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert empty ct on A"); + + for i in 0..N { + db_a.commitment_tree_insert( + EMPTY_PATH, + b"pool", + test_cmx(i + 1), + test_rho(i + 1), + test_ciphertext(i + 1), + None, + grove_version, + ) + .unwrap() + .expect("insert leaf on A"); + } + + let root_a = db_a + .root_hash(None, grove_version) + .unwrap() + .expect("source root_hash"); + + // --- Compute combined_root by re-opening CommitmentTree on A's storage --- + let pool_segments: [&[u8]; 1] = [b"pool"]; + let subtree_path = SubtreePath::from(pool_segments.as_slice()); + let prefix: [u8; 32] = RocksDbStorage::build_prefix(subtree_path.clone()).value; + + let tx_a = db_a.start_transaction(); + let ctx_open = db_a + .raw_storage() + .get_transactional_storage_context(subtree_path.clone(), None, &tx_a) + .unwrap(); + let ct_a = CommitmentTree::<_, DashMemo>::open(u64::from(N), TEST_CHUNK_POWER, ctx_open) + .value + .expect("CommitmentTree::open on A"); + let combined_root = ct_a + .compute_current_state_root() + .expect("compute_current_state_root on A"); + drop(ct_a); + + // --- Dump A's subtree to an SST (re-prepending the prefix that raw_iter strips) --- + let pool_segments_iter: [&[u8]; 1] = [b"pool"]; + let iter_path = SubtreePath::from(pool_segments_iter.as_slice()); + let iter_ctx = db_a + .raw_storage() + .get_transactional_storage_context(iter_path, None, &tx_a) + .unwrap(); + let sst_tmp = TempDir::new().expect("sst tempdir"); + let sst_path = sst_tmp.path().join("dump.sst"); + + let opts = Options::default(); + let mut sst = SstFileWriter::create(&opts); + sst.open(&sst_path).expect("SstFileWriter::open"); + + let mut iter = iter_ctx.raw_iter(); + iter.seek_to_first().unwrap(); + let mut key_count: u64 = 0; + while iter.valid().unwrap() { + let user_key = iter + .key() + .unwrap() + .expect("iter.key() during dump") + .to_vec(); + let value = iter + .value() + .unwrap() + .expect("iter.value() during dump") + .to_vec(); + let mut full_key = Vec::with_capacity(32 + user_key.len()); + full_key.extend_from_slice(&prefix); + full_key.extend_from_slice(&user_key); + sst.put(&full_key, &value).expect("SstFileWriter::put"); + key_count += 1; + iter.next().unwrap(); + } + sst.finish().expect("SstFileWriter::finish"); + assert!(key_count > 0, "dumped SST must contain at least one key"); + drop(iter); + drop(iter_ctx); + drop(tx_a); + drop(db_a); + + // --- B: fresh GroveDb with same empty commitment_tree skeleton --- + let db_b = make_empty_grovedb(); + db_b.insert( + EMPTY_PATH, + b"pool", + Element::empty_commitment_tree(TEST_CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert empty ct on B"); + + db_b.ingest_subtree_sst("default", &sst_path) + .expect("ingest_subtree_sst"); + + let restored_element = Element::new_commitment_tree(u64::from(N), TEST_CHUNK_POWER, None); + db_b.replace_subtree_root( + EMPTY_PATH, + b"pool", + restored_element, + combined_root, + None, + grove_version, + ) + .unwrap() + .expect("replace_subtree_root"); + + let root_b = db_b + .root_hash(None, grove_version) + .unwrap() + .expect("restored root_hash"); + + assert_eq!( + root_a, root_b, + "post-restore GroveDB root_hash must match source byte-for-byte" + ); +} + +#[cfg(feature = "unsafe-dump-load")] +#[test] +fn replace_subtree_root_rejects_non_tree_element() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"pool", + Element::empty_commitment_tree(TEST_CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert ct"); + + let item = Element::new_item(b"not a tree".to_vec()); + let result = db + .replace_subtree_root(EMPTY_PATH, b"pool", item, [0u8; 32], None, grove_version) + .unwrap(); + assert!( + matches!(result, Err(Error::InvalidInput(_))), + "expected InvalidInput for non-tree element, got {result:?}" + ); +}