From 595b30ca4047624e31514943e525261839ae5d94 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 26 Feb 2026 16:00:05 +0700 Subject: [PATCH 1/6] feat: add grovedb-commitment-tree crate Orchard-style commitment tree integration for GroveDB, combining a Sinsemilla frontier (for ZK-friendly anchor computation) with a BulkAppendTree (for efficient append-only storage with epoch compaction). Key components: - CommitmentFrontier: depth-32 incremental Merkle tree using MerkleHashOrchard (Sinsemilla) hashing, ~1KB constant size - CommitmentTree: server-side tree generic over MemoSize, with typed ciphertext append and payload size validation - ClientMemoryCommitmentTree: in-memory client for witness generation - ClientPersistentCommitmentTree: SQLite-backed persistent client - Ciphertext serialization helpers (serialize/deserialize/size) Also adds sinsemilla_hash_calls field to OperationCost for tracking elliptic curve hash operations separately from Blake3 node hashes. 72 tests covering frontier, storage, client, and SQLite operations. Co-Authored-By: Claude Opus 4.6 --- Cargo.toml | 1 + costs/src/lib.rs | 7 + grovedb-commitment-tree/Cargo.toml | 37 + .../benches/verification.rs | 150 ++++ .../client/client_memory_commitment_tree.rs | 123 +++ .../client_persistent_commitment_tree.rs | 170 ++++ grovedb-commitment-tree/src/client/mod.rs | 31 + .../src/client/sqlite_client_tests.rs | 193 +++++ .../src/client/sqlite_store.rs | 738 ++++++++++++++++++ .../src/client/sqlite_store_tests.rs | 444 +++++++++++ grovedb-commitment-tree/src/client/tests.rs | 242 ++++++ .../src/commitment_frontier/mod.rs | 230 ++++++ .../src/commitment_frontier/tests.rs | 318 ++++++++ .../src/commitment_tree/mod.rs | 353 +++++++++ .../src/commitment_tree/tests.rs | 728 +++++++++++++++++ grovedb-commitment-tree/src/error.rs | 15 + grovedb-commitment-tree/src/lib.rs | 107 +++ 17 files changed, 3887 insertions(+) create mode 100644 grovedb-commitment-tree/Cargo.toml create mode 100644 grovedb-commitment-tree/benches/verification.rs create mode 100644 grovedb-commitment-tree/src/client/client_memory_commitment_tree.rs create mode 100644 grovedb-commitment-tree/src/client/client_persistent_commitment_tree.rs create mode 100644 grovedb-commitment-tree/src/client/mod.rs create mode 100644 grovedb-commitment-tree/src/client/sqlite_client_tests.rs create mode 100644 grovedb-commitment-tree/src/client/sqlite_store.rs create mode 100644 grovedb-commitment-tree/src/client/sqlite_store_tests.rs create mode 100644 grovedb-commitment-tree/src/client/tests.rs create mode 100644 grovedb-commitment-tree/src/commitment_frontier/mod.rs create mode 100644 grovedb-commitment-tree/src/commitment_frontier/tests.rs create mode 100644 grovedb-commitment-tree/src/commitment_tree/mod.rs create mode 100644 grovedb-commitment-tree/src/commitment_tree/tests.rs create mode 100644 grovedb-commitment-tree/src/error.rs create mode 100644 grovedb-commitment-tree/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 77d0855f4..0591cddd2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,5 +16,6 @@ members = [ "grovedb-merkle-mountain-range", "grovedb-dense-fixed-sized-merkle-tree", "grovedb-bulk-append-tree", + "grovedb-commitment-tree", "grovedb-query", ] diff --git a/costs/src/lib.rs b/costs/src/lib.rs index 7e576dc1b..86cbd1a23 100644 --- a/costs/src/lib.rs +++ b/costs/src/lib.rs @@ -110,6 +110,10 @@ pub struct OperationCost { pub storage_loaded_bytes: u64, /// How many times node hashing was done (for merkelized tree). pub hash_node_calls: u32, + /// How many Sinsemilla hash operations were done (elliptic curve operations + /// for commitment tree anchors). These are significantly more expensive + /// than Blake3 node hashes. + pub sinsemilla_hash_calls: u32, } impl OperationCost { @@ -176,6 +180,7 @@ impl OperationCost { && self.storage_cost.worse_or_eq_than(&other.storage_cost) && self.storage_loaded_bytes >= other.storage_loaded_bytes && self.hash_node_calls >= other.hash_node_calls + && self.sinsemilla_hash_calls >= other.sinsemilla_hash_calls } /// add storage_cost costs for key and value storages @@ -291,6 +296,7 @@ impl Add for OperationCost { storage_cost: self.storage_cost + rhs.storage_cost, storage_loaded_bytes: self.storage_loaded_bytes + rhs.storage_loaded_bytes, hash_node_calls: self.hash_node_calls + rhs.hash_node_calls, + sinsemilla_hash_calls: self.sinsemilla_hash_calls + rhs.sinsemilla_hash_calls, } } } @@ -301,6 +307,7 @@ impl AddAssign for OperationCost { self.storage_cost += rhs.storage_cost; self.storage_loaded_bytes += rhs.storage_loaded_bytes; self.hash_node_calls += rhs.hash_node_calls; + self.sinsemilla_hash_calls += rhs.sinsemilla_hash_calls; } } diff --git a/grovedb-commitment-tree/Cargo.toml b/grovedb-commitment-tree/Cargo.toml new file mode 100644 index 000000000..2d7c748e2 --- /dev/null +++ b/grovedb-commitment-tree/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "grovedb-commitment-tree" +description = "Orchard-style commitment tree integration for GroveDB" +version = "4.0.0" +authors = ["Samuel Westrich "] +edition = "2021" +license = "MIT" +homepage = "https://www.grovedb.org" +repository = "https://github.com/dashpay/grovedb" +readme = "../README.md" +documentation = "https://docs.rs/grovedb" + +[features] +default = [] +server = [] +storage = ["grovedb-storage", "grovedb-bulk-append-tree", "server"] +client = ["shardtree"] +sqlite = ["shardtree", "rusqlite"] + +[dependencies] +orchard = { git = "https://github.com/dashpay/orchard.git", rev = "41c8f7169f2683c99cf0e0c63e8d25ec12c47a79", features = ["circuit"] } +incrementalmerkletree = "0.8" +shardtree = { version = "0.6", optional = true } +rusqlite = { version = "0.38", features = ["bundled"], optional = true } +grovedb-costs = { version = "4.0.0", path = "../costs" } +grovedb-storage = { version = "4.0.0", path = "../storage", optional = true } +grovedb-bulk-append-tree = { version = "4.0.0", path = "../grovedb-bulk-append-tree", optional = true } +thiserror = "2.0" + +[dev-dependencies] +tempfile = "3" +criterion = "0.4" +rand = "0.8" + +[[bench]] +name = "verification" +harness = false diff --git a/grovedb-commitment-tree/benches/verification.rs b/grovedb-commitment-tree/benches/verification.rs new file mode 100644 index 000000000..2334acffa --- /dev/null +++ b/grovedb-commitment-tree/benches/verification.rs @@ -0,0 +1,150 @@ +//! Benchmarks for Orchard proof and signature verification. +//! +//! Measures the actual time ratios between: +//! - Halo 2 ZK proof verification (per-bundle, scales with action count) +//! - RedPallas spend auth signature verification (per-action) +//! - RedPallas binding signature verification (per-bundle) +//! - Full BatchValidator (proof + all signatures) +//! +//! Run with: +//! ``` +//! cargo bench -p grovedb-commitment-tree --bench verification +//! ``` +//! +//! NOTE: The first run takes ~35 seconds to build ProvingKey + VerifyingKey. + +use std::sync::OnceLock; + +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +use grovedb_commitment_tree::{ + Anchor, Authorized, BatchValidator, Builder, Bundle, BundleType, DashMemo, Flags, + FullViewingKey, Hashable, MerkleHashOrchard, NoteValue, ProvingKey, Scope, SpendingKey, + VerifyingKey, +}; +use rand::rngs::OsRng; + +static PROVING_KEY: OnceLock = OnceLock::new(); +static VERIFYING_KEY: OnceLock = OnceLock::new(); + +fn get_pk() -> &'static ProvingKey { + PROVING_KEY.get_or_init(ProvingKey::build) +} + +fn get_vk() -> &'static VerifyingKey { + VERIFYING_KEY.get_or_init(VerifyingKey::build) +} + +/// Build a shielding bundle (output-only, SPENDS_DISABLED) with N outputs = N +/// actions. +fn build_bundle(num_outputs: usize) -> Bundle { + let mut rng = OsRng; + let pk = get_pk(); + + let sk = SpendingKey::from_bytes([7; 32]).unwrap(); + let fvk = FullViewingKey::from(&sk); + let recipient = fvk.address_at(0u32, Scope::External); + + let anchor: Anchor = MerkleHashOrchard::empty_root(32.into()).into(); + + let mut builder = Builder::::new( + BundleType::Transactional { + flags: Flags::SPENDS_DISABLED, + bundle_required: false, + }, + anchor, + ); + + for _ in 0..num_outputs { + builder + .add_output(None, recipient, NoteValue::from_raw(5000), [0u8; 36]) + .unwrap(); + } + + let (unauthorized, _) = builder.build::(&mut rng).unwrap().unwrap(); + let sighash: [u8; 32] = unauthorized.commitment().into(); + let proven = unauthorized.create_proof(pk, &mut rng).unwrap(); + proven.apply_signatures(rng, sighash, &[]).unwrap() +} + +fn benchmark_proof_verification(c: &mut Criterion) { + let vk = get_vk(); + + // Pre-build bundles with 1-4 actions + let bundles: Vec<_> = (1..=4).map(build_bundle).collect(); + + // --- Halo 2 proof verification (scales with action count) --- + { + let mut group = c.benchmark_group("halo2_proof"); + group.sample_size(10); + for (i, bundle) in bundles.iter().enumerate() { + let num_actions = i + 1; + let instances: Vec<_> = bundle + .actions() + .iter() + .map(|a| a.to_instance(*bundle.flags(), *bundle.anchor())) + .collect(); + + group.bench_function(BenchmarkId::new("actions", num_actions), |b| { + b.iter(|| bundle.authorization().proof().verify(vk, &instances)); + }); + } + group.finish(); + } +} + +fn benchmark_signature_verification(c: &mut Criterion) { + let bundle = build_bundle(1); + let sighash: [u8; 32] = bundle.commitment().into(); + + // --- RedPallas spend auth signature (per-action cost) --- + { + let action = &bundle.actions()[0]; + let rk = action.rk(); + let sig = action.authorization(); + + c.bench_function("redpallas_spend_auth_sig", |b| { + b.iter(|| rk.verify(&sighash, sig)); + }); + } + + // --- RedPallas binding signature (per-bundle cost) --- + { + let bvk = bundle.binding_validating_key(); + let binding_sig = bundle.authorization().binding_signature(); + + c.bench_function("redpallas_binding_sig", |b| { + b.iter(|| bvk.verify(&sighash, binding_sig)); + }); + } +} + +fn benchmark_batch_validator(c: &mut Criterion) { + let vk = get_vk(); + + let bundles: Vec<_> = (1..=4).map(build_bundle).collect(); + + // --- Full BatchValidator: proof + spend auth sigs + binding sig --- + let mut group = c.benchmark_group("batch_validator"); + group.sample_size(10); + for (i, bundle) in bundles.iter().enumerate() { + let num_actions = i + 1; + let sighash: [u8; 32] = bundle.commitment().into(); + + group.bench_function(BenchmarkId::new("actions", num_actions), |b| { + b.iter(|| { + let mut validator = BatchValidator::new(); + validator.add_bundle(bundle, sighash); + validator.validate(vk, OsRng) + }); + }); + } + group.finish(); +} + +criterion_group!( + benches, + benchmark_proof_verification, + benchmark_signature_verification, + benchmark_batch_validator, +); +criterion_main!(benches); diff --git a/grovedb-commitment-tree/src/client/client_memory_commitment_tree.rs b/grovedb-commitment-tree/src/client/client_memory_commitment_tree.rs new file mode 100644 index 000000000..4d91fe010 --- /dev/null +++ b/grovedb-commitment-tree/src/client/client_memory_commitment_tree.rs @@ -0,0 +1,123 @@ +use incrementalmerkletree::{Position, Retention}; +use orchard::{ + tree::{MerkleHashOrchard, MerklePath}, + Anchor, NOTE_COMMITMENT_TREE_DEPTH, +}; +use shardtree::{store::memory::MemoryShardStore, ShardTree}; + +use crate::commitment_frontier::{merkle_hash_from_bytes, CommitmentTreeError}; + +/// Shard height for the ShardTree. Each shard covers 16 levels. +const SHARD_HEIGHT: u8 = 4; + +/// Client-side Orchard commitment tree with full Merkle witness support. +/// +/// Wraps `ShardTree, 32, 4>` with +/// a convenient Orchard-typed API. All state is in-memory and lost on drop. +/// +/// Use this for: +/// - Wallet note tracking and spend witness generation +/// - Test harnesses that construct valid Orchard spend bundles +/// +/// Do **not** use this for server-side anchor tracking — use +/// [`CommitmentFrontier`](crate::commitment_frontier::CommitmentFrontier) +/// instead. +pub struct ClientMemoryCommitmentTree { + inner: ShardTree< + MemoryShardStore, + { NOTE_COMMITMENT_TREE_DEPTH as u8 }, + SHARD_HEIGHT, + >, +} + +impl ClientMemoryCommitmentTree { + /// Create a new empty client commitment tree. + /// + /// `max_checkpoints` controls how many checkpoints are retained before + /// the oldest are pruned. + pub fn new(max_checkpoints: usize) -> Self { + Self { + inner: ShardTree::new(MemoryShardStore::empty(), max_checkpoints), + } + } + + /// Append a note commitment to the tree. + /// + /// `cmx` is the 32-byte extracted note commitment. `retention` controls + /// whether the leaf is marked for witness generation, checkpointed, or + /// ephemeral. + pub fn append( + &mut self, + cmx: [u8; 32], + retention: Retention, + ) -> Result<(), CommitmentTreeError> { + let leaf = merkle_hash_from_bytes(&cmx).ok_or(CommitmentTreeError::InvalidFieldElement)?; + self.inner + .batch_insert(self.next_position()?, std::iter::once((leaf, retention))) + .map_err(|e| CommitmentTreeError::InvalidData(format!("append failed: {e}")))?; + Ok(()) + } + + /// Create a checkpoint at the current tree state. + /// + /// Checkpoints allow `witness_at_checkpoint_depth` to produce witnesses + /// relative to historical anchors. + pub fn checkpoint(&mut self, checkpoint_id: u32) -> Result { + self.inner + .checkpoint(checkpoint_id) + .map_err(|e| CommitmentTreeError::InvalidData(format!("checkpoint failed: {e}"))) + } + + /// Get the position of the most recently appended leaf. + /// + /// Returns `None` if the tree is empty. + pub fn max_leaf_position(&self) -> Result, CommitmentTreeError> { + self.inner + .max_leaf_position(None) + .map_err(|e| CommitmentTreeError::InvalidData(format!("max_leaf_position failed: {e}"))) + } + + /// Generate a Merkle witness (authentication path) for spending a note + /// at the given position. + /// + /// `checkpoint_depth` is 0 for the current tree state, 1 for the + /// previous checkpoint, etc. The leaf at `position` must have been + /// inserted with `Retention::Marked` or `Retention::Checkpoint { marking: + /// Marking::Marked, .. }`. + pub fn witness( + &self, + position: Position, + checkpoint_depth: usize, + ) -> Result, CommitmentTreeError> { + self.inner + .witness_at_checkpoint_depth(position, checkpoint_depth) + .map(|opt| opt.map(MerklePath::from)) + .map_err(|e| CommitmentTreeError::InvalidData(format!("witness failed: {e}"))) + } + + /// Get the current root as an Orchard `Anchor`. + /// + /// Returns the empty tree anchor if no leaves have been appended. + pub fn anchor(&self) -> Result { + match self + .inner + .root_at_checkpoint_depth(None) + .map_err(|e| CommitmentTreeError::InvalidData(format!("root failed: {e}")))? + { + Some(root) => Ok(Anchor::from(root)), + None => Ok(Anchor::empty_tree()), + } + } + + /// Get the next insertion position (0 for empty tree). + fn next_position(&self) -> Result { + let pos = self + .inner + .max_leaf_position(None) + .map_err(|e| CommitmentTreeError::InvalidData(format!("max_leaf_position: {e}")))?; + Ok(match pos { + Some(p) => p + 1, + None => Position::from(0), + }) + } +} diff --git a/grovedb-commitment-tree/src/client/client_persistent_commitment_tree.rs b/grovedb-commitment-tree/src/client/client_persistent_commitment_tree.rs new file mode 100644 index 000000000..0ebcafd76 --- /dev/null +++ b/grovedb-commitment-tree/src/client/client_persistent_commitment_tree.rs @@ -0,0 +1,170 @@ +//! Persistent client commitment tree backed by SQLite. +//! +//! This module provides [`ClientPersistentCommitmentTree`], a commitment tree +//! that persists its state in a SQLite database. The tree survives application +//! restarts and can be re-opened from the same database. +//! +//! # Bring-your-own-connection +//! +//! You can pass **any** `rusqlite::Connection` — for example, your wallet's +//! existing database. The store only creates its own tables (prefixed with +//! `commitment_tree_`) and will not interfere with other tables. +//! +//! ```ignore +//! use rusqlite::Connection; +//! use grovedb_commitment_tree::ClientPersistentCommitmentTree; +//! +//! // Use your existing wallet database +//! let conn = Connection::open("wallet.db")?; +//! let mut tree = ClientPersistentCommitmentTree::open(conn, 100)?; +//! tree.append(cmx_bytes, Retention::Marked)?; +//! // State is persisted — survives restarts. +//! ``` + +use std::{ + path::Path, + sync::{Arc, Mutex}, +}; + +use incrementalmerkletree::{Position, Retention}; +use orchard::{ + tree::{Anchor, MerklePath}, + NOTE_COMMITMENT_TREE_DEPTH, +}; +use rusqlite::Connection; +use shardtree::ShardTree; + +use super::sqlite_store::{SqliteShardStore, SqliteShardStoreError}; +use crate::commitment_frontier::{merkle_hash_from_bytes, CommitmentTreeError}; + +/// Shard height for the ShardTree. Each shard covers 16 levels. +const SHARD_HEIGHT: u8 = 4; + +/// Persistent Orchard commitment tree backed by SQLite. +/// +/// Same API as +/// [`ClientMemoryCommitmentTree`](crate::ClientMemoryCommitmentTree) +/// but all state is persisted to a SQLite database. Drop and re-open from the +/// same database to resume where you left off. +pub struct ClientPersistentCommitmentTree { + inner: ShardTree, +} + +impl ClientPersistentCommitmentTree { + /// Open a persistent commitment tree using an existing SQLite connection. + /// + /// The required tables are created automatically if they don't exist. + /// Pass your wallet's existing database connection to share the same file. + pub fn open(conn: Connection, max_checkpoints: usize) -> Result { + let store = SqliteShardStore::new(conn)?; + Ok(Self { + inner: ShardTree::new(store, max_checkpoints), + }) + } + + /// Open a persistent commitment tree on a shared SQLite connection. + /// + /// Use this when your application already holds an `Arc>` + /// (e.g., a wallet database). The commitment tree tables are created if + /// missing, and the mutex is locked only for the duration of each SQL + /// operation. + pub fn open_on_shared_connection( + conn: Arc>, + max_checkpoints: usize, + ) -> Result { + let store = SqliteShardStore::new_shared(conn)?; + Ok(Self { + inner: ShardTree::new(store, max_checkpoints), + }) + } + + /// Open a persistent commitment tree at the given file path. + /// + /// Creates the SQLite database if it doesn't exist. This is a convenience + /// method for applications that want a dedicated commitment tree database. + pub fn open_path( + path: impl AsRef, + max_checkpoints: usize, + ) -> Result { + let conn = Connection::open(path)?; + Self::open(conn, max_checkpoints) + } + + /// Append a note commitment to the tree. + /// + /// `cmx` is the 32-byte extracted note commitment. `retention` controls + /// whether the leaf is marked for witness generation, checkpointed, or + /// ephemeral. + pub fn append( + &mut self, + cmx: [u8; 32], + retention: Retention, + ) -> Result<(), CommitmentTreeError> { + let leaf = merkle_hash_from_bytes(&cmx).ok_or(CommitmentTreeError::InvalidFieldElement)?; + self.inner + .batch_insert(self.next_position()?, std::iter::once((leaf, retention))) + .map_err(|e| CommitmentTreeError::InvalidData(format!("append failed: {e}")))?; + Ok(()) + } + + /// Create a checkpoint at the current tree state. + /// + /// Checkpoints allow `witness_at_checkpoint_depth` to produce witnesses + /// relative to historical anchors. + pub fn checkpoint(&mut self, checkpoint_id: u32) -> Result { + self.inner + .checkpoint(checkpoint_id) + .map_err(|e| CommitmentTreeError::InvalidData(format!("checkpoint failed: {e}"))) + } + + /// Get the position of the most recently appended leaf. + /// + /// Returns `None` if the tree is empty. + pub fn max_leaf_position(&self) -> Result, CommitmentTreeError> { + self.inner + .max_leaf_position(None) + .map_err(|e| CommitmentTreeError::InvalidData(format!("max_leaf_position failed: {e}"))) + } + + /// Generate a Merkle witness (authentication path) for spending a note + /// at the given position. + /// + /// `checkpoint_depth` is 0 for the current tree state, 1 for the + /// previous checkpoint, etc. + pub fn witness( + &self, + position: Position, + checkpoint_depth: usize, + ) -> Result, CommitmentTreeError> { + self.inner + .witness_at_checkpoint_depth(position, checkpoint_depth) + .map(|opt| opt.map(MerklePath::from)) + .map_err(|e| CommitmentTreeError::InvalidData(format!("witness failed: {e}"))) + } + + /// Get the current root as an Orchard `Anchor`. + /// + /// Returns the empty tree anchor if no leaves have been appended. + pub fn anchor(&self) -> Result { + match self + .inner + .root_at_checkpoint_depth(None) + .map_err(|e| CommitmentTreeError::InvalidData(format!("root failed: {e}")))? + { + Some(root) => Ok(Anchor::from(root)), + None => Ok(Anchor::empty_tree()), + } + } + + /// Get the next insertion position (0 for empty tree). + fn next_position(&self) -> Result { + let pos = self + .inner + .max_leaf_position(None) + .map_err(|e| CommitmentTreeError::InvalidData(format!("max_leaf_position: {e}")))?; + Ok(match pos { + Some(p) => p + 1, + None => Position::from(0), + }) + } +} diff --git a/grovedb-commitment-tree/src/client/mod.rs b/grovedb-commitment-tree/src/client/mod.rs new file mode 100644 index 000000000..4bcbc00e7 --- /dev/null +++ b/grovedb-commitment-tree/src/client/mod.rs @@ -0,0 +1,31 @@ +//! Client-side commitment tree with full witness generation. +//! +//! This module provides [`ClientMemoryCommitmentTree`], a wrapper around +//! `shardtree::ShardTree` with an in-memory store, pinned to Orchard types. +//! It is intended for wallets and test harnesses that need to generate +//! Merkle path witnesses for spending notes. +//! +//! Enable the `client` feature to use this module: +//! ```toml +//! grovedb-commitment-tree = { version = "4", features = ["client"] } +//! ``` + +mod client_memory_commitment_tree; +pub use client_memory_commitment_tree::ClientMemoryCommitmentTree; + +#[cfg(feature = "sqlite")] +mod sqlite_store; +#[cfg(feature = "sqlite")] +pub use sqlite_store::{SqliteShardStore, SqliteShardStoreError}; + +#[cfg(feature = "sqlite")] +mod client_persistent_commitment_tree; +#[cfg(feature = "sqlite")] +pub use client_persistent_commitment_tree::ClientPersistentCommitmentTree; + +#[cfg(all(test, feature = "sqlite"))] +mod sqlite_client_tests; +#[cfg(all(test, feature = "sqlite"))] +mod sqlite_store_tests; +#[cfg(test)] +mod tests; diff --git a/grovedb-commitment-tree/src/client/sqlite_client_tests.rs b/grovedb-commitment-tree/src/client/sqlite_client_tests.rs new file mode 100644 index 000000000..bc31cf703 --- /dev/null +++ b/grovedb-commitment-tree/src/client/sqlite_client_tests.rs @@ -0,0 +1,193 @@ +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use incrementalmerkletree::{Hashable, Level, Position, Retention}; + use orchard::tree::{Anchor, MerkleHashOrchard}; + use rusqlite::Connection; + + use crate::ClientPersistentCommitmentTree; + + fn test_leaf(index: u64) -> [u8; 32] { + let empty = MerkleHashOrchard::empty_leaf(); + let varied = + MerkleHashOrchard::combine(Level::from((index % 31) as u8 + 1), &empty, &empty); + MerkleHashOrchard::combine(Level::from(0), &empty, &varied).to_bytes() + } + + fn memory_tree() -> ClientPersistentCommitmentTree { + let conn = Connection::open_in_memory().expect("open in-memory sqlite"); + ClientPersistentCommitmentTree::open(conn, 100).expect("open tree") + } + + #[test] + fn test_empty_tree() { + let tree = memory_tree(); + assert_eq!(tree.max_leaf_position().expect("max_leaf_position"), None); + assert_eq!(tree.anchor().expect("anchor"), Anchor::empty_tree()); + } + + #[test] + fn test_append_and_position() { + let mut tree = memory_tree(); + + tree.append(test_leaf(0), Retention::Marked) + .expect("append 0"); + assert_eq!( + tree.max_leaf_position().expect("pos"), + Some(Position::from(0)) + ); + + tree.append(test_leaf(1), Retention::Ephemeral) + .expect("append 1"); + assert_eq!( + tree.max_leaf_position().expect("pos"), + Some(Position::from(1)) + ); + } + + #[test] + fn test_anchor_changes() { + let mut tree = memory_tree(); + let empty_anchor = tree.anchor().expect("anchor"); + + tree.append(test_leaf(0), Retention::Marked) + .expect("append 0"); + let anchor1 = tree.anchor().expect("anchor"); + assert_ne!(empty_anchor, anchor1); + + tree.append(test_leaf(1), Retention::Marked) + .expect("append 1"); + let anchor2 = tree.anchor().expect("anchor"); + assert_ne!(anchor1, anchor2); + } + + #[test] + fn test_witness_generation() { + let mut tree = memory_tree(); + + tree.append(test_leaf(0), Retention::Marked) + .expect("append 0"); + tree.append(test_leaf(1), Retention::Ephemeral) + .expect("append 1"); + tree.checkpoint(1).expect("checkpoint"); + + let path = tree.witness(Position::from(0), 0).expect("witness"); + assert!(path.is_some(), "should produce witness for marked leaf"); + } + + #[test] + fn test_persistence_across_reopen() { + // Use a temp file so we can reopen it + let dir = tempfile::tempdir().expect("create temp dir"); + let db_path = dir.path().join("test_commitment.db"); + + // Phase 1: create tree, append leaves, get anchor + let anchor_before; + let position_before; + { + let mut tree = + ClientPersistentCommitmentTree::open_path(&db_path, 100).expect("open tree"); + for i in 0..20u64 { + tree.append(test_leaf(i), Retention::Marked) + .expect("append"); + } + tree.checkpoint(1).expect("checkpoint"); + anchor_before = tree.anchor().expect("anchor"); + position_before = tree.max_leaf_position().expect("position"); + // tree is dropped here, connection closed + } + + // Phase 2: reopen from same file, verify state matches + { + let tree = + ClientPersistentCommitmentTree::open_path(&db_path, 100).expect("reopen tree"); + let anchor_after = tree.anchor().expect("anchor"); + let position_after = tree.max_leaf_position().expect("position"); + + assert_eq!(anchor_before, anchor_after, "anchor should survive restart"); + assert_eq!( + position_before, position_after, + "position should survive restart" + ); + } + } + + #[test] + fn test_bring_your_own_connection() { + // Verify the store coexists with other tables + let conn = Connection::open_in_memory().expect("open sqlite"); + conn.execute( + "CREATE TABLE my_app_data (id INTEGER PRIMARY KEY, value TEXT)", + [], + ) + .expect("create app table"); + conn.execute( + "INSERT INTO my_app_data (id, value) VALUES (1, 'hello')", + [], + ) + .expect("insert app data"); + + let mut tree = ClientPersistentCommitmentTree::open(conn, 100).expect("open tree"); + tree.append(test_leaf(0), Retention::Marked) + .expect("append"); + + // We can't directly query the connection since it's owned by the tree, + // but the fact that open() succeeded proves coexistence works. + } + + #[test] + fn test_witness_after_reopen() { + let dir = tempfile::tempdir().expect("create temp dir"); + let db_path = dir.path().join("test_witness_reopen.db"); + + // Phase 1: append a marked leaf and checkpoint + { + let mut tree = + ClientPersistentCommitmentTree::open_path(&db_path, 100).expect("open tree"); + tree.append(test_leaf(0), Retention::Marked) + .expect("append marked"); + tree.append(test_leaf(1), Retention::Ephemeral) + .expect("append ephemeral"); + tree.checkpoint(1).expect("checkpoint"); + } + + // Phase 2: reopen and generate witness + { + let tree = + ClientPersistentCommitmentTree::open_path(&db_path, 100).expect("reopen tree"); + let path = tree + .witness(Position::from(0), 0) + .expect("witness after reopen"); + assert!( + path.is_some(), + "should produce witness for marked leaf after reopen" + ); + } + } + + #[test] + fn test_shared_connection_append_and_anchor() { + let conn = Connection::open_in_memory().expect("open sqlite"); + let arc = Arc::new(Mutex::new(conn)); + + let mut tree = ClientPersistentCommitmentTree::open_on_shared_connection(arc.clone(), 100) + .expect("open shared tree"); + + let empty_anchor = tree.anchor().expect("anchor"); + + tree.append(test_leaf(0), Retention::Marked) + .expect("append via shared"); + let anchor1 = tree.anchor().expect("anchor"); + assert_ne!(empty_anchor, anchor1); + + // The Arc is still usable from outside + let guard = arc.lock().expect("lock"); + let count: i64 = guard + .query_row("SELECT COUNT(*) FROM commitment_tree_shards", [], |row| { + row.get(0) + }) + .expect("direct query"); + assert!(count > 0, "shards should have been written"); + } +} diff --git a/grovedb-commitment-tree/src/client/sqlite_store.rs b/grovedb-commitment-tree/src/client/sqlite_store.rs new file mode 100644 index 000000000..2d8cea56c --- /dev/null +++ b/grovedb-commitment-tree/src/client/sqlite_store.rs @@ -0,0 +1,738 @@ +//! SQLite-backed ShardStore for persistent commitment tree storage. +//! +//! Implements the `shardtree::store::ShardStore` trait using a SQLite database, +//! allowing commitment tree state to persist across application restarts. +//! +//! The store creates 4 tables with a `commitment_tree_` prefix so it can +//! coexist safely in any existing SQLite database. +//! +//! # Connection modes +//! +//! - **Owned**: `SqliteShardStore::new(conn)` takes ownership of a +//! `Connection`. +//! - **Shared**: `SqliteShardStore::new_shared(arc)` shares an +//! `Arc>` with other components (e.g., PMT's `Database`). + +use std::{ + collections::BTreeSet, + sync::{Arc, Mutex}, +}; + +use incrementalmerkletree::{Address, Level, Position}; +use orchard::tree::MerkleHashOrchard; +use rusqlite::{params, Connection, OptionalExtension}; +use shardtree::{ + store::{Checkpoint, ShardStore, TreeState}, + LocatedPrunableTree, LocatedTree, Node, PrunableTree, RetentionFlags, Tree, +}; + +use crate::commitment_frontier::merkle_hash_from_bytes; + +/// Shard height — must match the value used in +/// `ClientPersistentCommitmentTree`. +pub(crate) const SHARD_HEIGHT: u8 = 4; + +/// How the store accesses the SQLite connection. +enum ConnectionHolder { + /// The store owns the connection exclusively. + Owned(Connection), + /// The store shares the connection with other components. + Shared(Arc>), +} + +/// SQLite-backed implementation of `ShardStore` for Orchard commitment trees. +/// +/// Stores shard data, cap, and checkpoints in 4 SQLite tables prefixed with +/// `commitment_tree_`. The tables are created automatically on construction. +/// +/// # Connection modes +/// +/// Use [`new`](Self::new) with an owned `Connection`, or +/// [`new_shared`](Self::new_shared) with an `Arc>` to share +/// one connection with the rest of your application. +pub struct SqliteShardStore { + holder: ConnectionHolder, +} + +/// Errors from the SQLite shard store. +#[derive(Debug)] +pub enum SqliteShardStoreError { + /// An error from the underlying SQLite connection. + Sqlite(rusqlite::Error), + /// A serialization or deserialization error. + Serialization(String), +} + +impl std::fmt::Display for SqliteShardStoreError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Sqlite(e) => write!(f, "sqlite error: {e}"), + Self::Serialization(msg) => write!(f, "serialization error: {msg}"), + } + } +} + +impl std::error::Error for SqliteShardStoreError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Sqlite(e) => Some(e), + Self::Serialization(_) => None, + } + } +} + +impl From for SqliteShardStoreError { + fn from(e: rusqlite::Error) -> Self { + Self::Sqlite(e) + } +} + +impl SqliteShardStore { + /// Create a store that **owns** the given connection. + /// + /// Creates the required tables if they do not already exist. + pub fn new(conn: Connection) -> Result { + create_tables(&conn)?; + Ok(Self { + holder: ConnectionHolder::Owned(conn), + }) + } + + /// Create a store that **shares** a connection via + /// `Arc>`. + /// + /// This lets you use the same SQLite connection that the rest of your + /// application (e.g., a wallet database) already holds. The store locks the + /// mutex for each individual SQL operation. + /// + /// Creates the required tables if they do not already exist. + pub fn new_shared(conn: Arc>) -> Result { + { + let guard = conn.lock().expect("connection mutex poisoned"); + create_tables(&guard)?; + } + Ok(Self { + holder: ConnectionHolder::Shared(conn), + }) + } + + /// Execute a closure with a reference to the underlying connection. + /// + /// For the `Owned` variant this is a direct borrow. For `Shared` it + /// acquires the mutex for the duration of the closure. + pub(crate) fn with_conn(&self, f: impl FnOnce(&Connection) -> T) -> T { + match &self.holder { + ConnectionHolder::Owned(conn) => f(conn), + ConnectionHolder::Shared(arc) => { + let guard = arc.lock().expect("connection mutex poisoned"); + f(&guard) + } + } + } +} + +/// Create the 4 commitment-tree tables if they don't already exist. +fn create_tables(conn: &Connection) -> Result<(), SqliteShardStoreError> { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS commitment_tree_shards ( + shard_index INTEGER PRIMARY KEY, + shard_data BLOB NOT NULL + ); + CREATE TABLE IF NOT EXISTS commitment_tree_cap ( + id INTEGER PRIMARY KEY CHECK (id = 0), + cap_data BLOB NOT NULL + ); + CREATE TABLE IF NOT EXISTS commitment_tree_checkpoints ( + checkpoint_id INTEGER PRIMARY KEY, + position INTEGER + ); + CREATE TABLE IF NOT EXISTS commitment_tree_checkpoint_marks_removed ( + checkpoint_id INTEGER NOT NULL, + position INTEGER NOT NULL, + PRIMARY KEY (checkpoint_id, position), + FOREIGN KEY (checkpoint_id) REFERENCES commitment_tree_checkpoints(checkpoint_id) + );", + )?; + Ok(()) +} + +// --------------------------------------------------------------------------- +// SQL helpers — all take &Connection so they can be called from with_conn +// --------------------------------------------------------------------------- + +fn sql_get_shard( + conn: &Connection, + shard_root: Address, +) -> Result>, SqliteShardStoreError> { + let index = shard_root.index() as i64; + let row: Option> = conn + .query_row( + "SELECT shard_data FROM commitment_tree_shards WHERE shard_index = ?1", + params![index], + |row| row.get(0), + ) + .optional()?; + + match row { + None => Ok(None), + Some(data) => { + let mut pos = 0; + let tree = deserialize_tree(&data, &mut pos)?; + let located = LocatedTree::from_parts(shard_root, tree).map_err(|addr| { + SqliteShardStoreError::Serialization(format!( + "tree extends beyond shard root at {addr:?}" + )) + })?; + Ok(Some(located)) + } + } +} + +fn sql_last_shard( + conn: &Connection, +) -> Result>, SqliteShardStoreError> { + let row: Option<(i64, Vec)> = conn + .query_row( + "SELECT shard_index, shard_data FROM commitment_tree_shards ORDER BY shard_index DESC \ + LIMIT 1", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional()?; + + match row { + None => Ok(None), + Some((index, data)) => { + let addr = Address::from_parts(Level::from(SHARD_HEIGHT), index as u64); + let mut pos = 0; + let tree = deserialize_tree(&data, &mut pos)?; + let located = LocatedTree::from_parts(addr, tree).map_err(|addr| { + SqliteShardStoreError::Serialization(format!( + "tree extends beyond shard root at {addr:?}" + )) + })?; + Ok(Some(located)) + } + } +} + +fn sql_put_shard( + conn: &Connection, + subtree: &LocatedPrunableTree, +) -> Result<(), SqliteShardStoreError> { + let index = subtree.root_addr().index() as i64; + let data = serialize_tree(subtree.root()); + conn.execute( + "INSERT OR REPLACE INTO commitment_tree_shards (shard_index, shard_data) VALUES (?1, ?2)", + params![index, data], + )?; + Ok(()) +} + +fn sql_get_shard_roots(conn: &Connection) -> Result, SqliteShardStoreError> { + let mut stmt = + conn.prepare("SELECT shard_index FROM commitment_tree_shards ORDER BY shard_index")?; + let rows = stmt.query_map([], |row| { + let index: i64 = row.get(0)?; + Ok(Address::from_parts(Level::from(SHARD_HEIGHT), index as u64)) + })?; + let mut result = Vec::new(); + for addr in rows { + result.push(addr?); + } + Ok(result) +} + +fn sql_truncate_shards(conn: &Connection, shard_index: u64) -> Result<(), SqliteShardStoreError> { + conn.execute( + "DELETE FROM commitment_tree_shards WHERE shard_index >= ?1", + params![shard_index as i64], + )?; + Ok(()) +} + +fn sql_get_cap( + conn: &Connection, +) -> Result, SqliteShardStoreError> { + let row: Option> = conn + .query_row( + "SELECT cap_data FROM commitment_tree_cap WHERE id = 0", + [], + |row| row.get(0), + ) + .optional()?; + + match row { + None => Ok(Tree::empty()), + Some(data) => { + let mut pos = 0; + deserialize_tree(&data, &mut pos) + } + } +} + +fn sql_put_cap( + conn: &Connection, + cap: &PrunableTree, +) -> Result<(), SqliteShardStoreError> { + let data = serialize_tree(cap); + conn.execute( + "INSERT OR REPLACE INTO commitment_tree_cap (id, cap_data) VALUES (0, ?1)", + params![data], + )?; + Ok(()) +} + +fn sql_min_checkpoint_id(conn: &Connection) -> Result, SqliteShardStoreError> { + let row: Option = conn.query_row( + "SELECT MIN(checkpoint_id) FROM commitment_tree_checkpoints", + [], + |row| row.get::<_, Option>(0), + )?; + Ok(row) +} + +fn sql_max_checkpoint_id(conn: &Connection) -> Result, SqliteShardStoreError> { + let row: Option = conn.query_row( + "SELECT MAX(checkpoint_id) FROM commitment_tree_checkpoints", + [], + |row| row.get::<_, Option>(0), + )?; + Ok(row) +} + +fn sql_add_checkpoint( + conn: &Connection, + checkpoint_id: u32, + checkpoint: &Checkpoint, +) -> Result<(), SqliteShardStoreError> { + let position: Option = match checkpoint.tree_state() { + TreeState::Empty => None, + TreeState::AtPosition(pos) => Some(u64::from(pos) as i64), + }; + conn.execute( + "INSERT INTO commitment_tree_checkpoints (checkpoint_id, position) VALUES (?1, ?2)", + params![checkpoint_id, position], + )?; + + for mark_pos in checkpoint.marks_removed() { + conn.execute( + "INSERT INTO commitment_tree_checkpoint_marks_removed (checkpoint_id, position) \ + VALUES (?1, ?2)", + params![checkpoint_id, u64::from(*mark_pos) as i64], + )?; + } + Ok(()) +} + +fn sql_checkpoint_count(conn: &Connection) -> Result { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM commitment_tree_checkpoints", + [], + |row| row.get(0), + )?; + Ok(count as usize) +} + +fn sql_get_checkpoint_at_depth( + conn: &Connection, + checkpoint_depth: usize, +) -> Result, SqliteShardStoreError> { + let row: Option<(u32, Option)> = conn + .query_row( + "SELECT checkpoint_id, position FROM commitment_tree_checkpoints ORDER BY \ + checkpoint_id DESC LIMIT 1 OFFSET ?1", + params![checkpoint_depth as i64], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional()?; + + match row { + None => Ok(None), + Some((id, pos)) => { + let checkpoint = sql_load_checkpoint(conn, id, pos)?; + Ok(Some((id, checkpoint))) + } + } +} + +fn sql_get_checkpoint( + conn: &Connection, + checkpoint_id: u32, +) -> Result, SqliteShardStoreError> { + let row: Option> = conn + .query_row( + "SELECT position FROM commitment_tree_checkpoints WHERE checkpoint_id = ?1", + params![checkpoint_id], + |row| row.get(0), + ) + .optional()?; + + match row { + None => Ok(None), + Some(pos) => { + let checkpoint = sql_load_checkpoint(conn, checkpoint_id, pos)?; + Ok(Some(checkpoint)) + } + } +} + +fn sql_list_checkpoints( + conn: &Connection, + limit: usize, +) -> Result, SqliteShardStoreError> { + let mut stmt = conn.prepare( + "SELECT checkpoint_id, position FROM commitment_tree_checkpoints ORDER BY checkpoint_id \ + DESC LIMIT ?1", + )?; + let rows: Vec<(u32, Option)> = stmt + .query_map(params![limit as i64], |row| Ok((row.get(0)?, row.get(1)?)))? + .collect::, _>>()?; + + let mut result = Vec::with_capacity(rows.len()); + for (id, pos) in rows { + let checkpoint = sql_load_checkpoint(conn, id, pos)?; + result.push((id, checkpoint)); + } + Ok(result) +} + +fn sql_update_checkpoint_with( + conn: &Connection, + checkpoint_id: u32, + update: F, +) -> Result +where + F: Fn(&mut Checkpoint) -> Result<(), SqliteShardStoreError>, +{ + let existing = sql_get_checkpoint(conn, checkpoint_id)?; + match existing { + None => Ok(false), + Some(mut cp) => { + update(&mut cp)?; + conn.execute( + "DELETE FROM commitment_tree_checkpoint_marks_removed WHERE checkpoint_id = ?1", + params![checkpoint_id], + )?; + let position: Option = match cp.tree_state() { + TreeState::Empty => None, + TreeState::AtPosition(pos) => Some(u64::from(pos) as i64), + }; + conn.execute( + "UPDATE commitment_tree_checkpoints SET position = ?1 WHERE checkpoint_id = ?2", + params![position, checkpoint_id], + )?; + for mark_pos in cp.marks_removed() { + conn.execute( + "INSERT INTO commitment_tree_checkpoint_marks_removed (checkpoint_id, \ + position) VALUES (?1, ?2)", + params![checkpoint_id, u64::from(*mark_pos) as i64], + )?; + } + Ok(true) + } + } +} + +fn sql_remove_checkpoint( + conn: &Connection, + checkpoint_id: u32, +) -> Result<(), SqliteShardStoreError> { + conn.execute( + "DELETE FROM commitment_tree_checkpoint_marks_removed WHERE checkpoint_id = ?1", + params![checkpoint_id], + )?; + conn.execute( + "DELETE FROM commitment_tree_checkpoints WHERE checkpoint_id = ?1", + params![checkpoint_id], + )?; + Ok(()) +} + +fn sql_truncate_checkpoints_retaining( + conn: &Connection, + checkpoint_id: u32, +) -> Result<(), SqliteShardStoreError> { + conn.execute( + "DELETE FROM commitment_tree_checkpoint_marks_removed WHERE checkpoint_id > ?1", + params![checkpoint_id], + )?; + conn.execute( + "DELETE FROM commitment_tree_checkpoints WHERE checkpoint_id > ?1", + params![checkpoint_id], + )?; + conn.execute( + "DELETE FROM commitment_tree_checkpoint_marks_removed WHERE checkpoint_id = ?1", + params![checkpoint_id], + )?; + Ok(()) +} + +/// Load a full Checkpoint (including marks_removed). +fn sql_load_checkpoint( + conn: &Connection, + checkpoint_id: u32, + position: Option, +) -> Result { + let tree_state = match position { + None => TreeState::Empty, + Some(p) => TreeState::AtPosition(Position::from(p as u64)), + }; + + let mut stmt = conn.prepare( + "SELECT position FROM commitment_tree_checkpoint_marks_removed WHERE checkpoint_id = ?1", + )?; + let marks: BTreeSet = stmt + .query_map(params![checkpoint_id], |row| { + let p: i64 = row.get(0)?; + Ok(Position::from(p as u64)) + })? + .collect::, _>>()?; + + Ok(Checkpoint::from_parts(tree_state, marks)) +} + +// --------------------------------------------------------------------------- +// ShardStore trait implementation — delegates to sql_* via with_conn +// --------------------------------------------------------------------------- + +impl ShardStore for SqliteShardStore { + type CheckpointId = u32; + type Error = SqliteShardStoreError; + type H = MerkleHashOrchard; + + fn get_shard( + &self, + shard_root: Address, + ) -> Result>, Self::Error> { + self.with_conn(|conn| sql_get_shard(conn, shard_root)) + } + + fn last_shard(&self) -> Result>, Self::Error> { + self.with_conn(sql_last_shard) + } + + fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { + self.with_conn(|conn| sql_put_shard(conn, &subtree)) + } + + fn get_shard_roots(&self) -> Result, Self::Error> { + self.with_conn(sql_get_shard_roots) + } + + fn truncate_shards(&mut self, shard_index: u64) -> Result<(), Self::Error> { + self.with_conn(|conn| sql_truncate_shards(conn, shard_index)) + } + + fn get_cap(&self) -> Result, Self::Error> { + self.with_conn(sql_get_cap) + } + + fn put_cap(&mut self, cap: PrunableTree) -> Result<(), Self::Error> { + self.with_conn(|conn| sql_put_cap(conn, &cap)) + } + + fn min_checkpoint_id(&self) -> Result, Self::Error> { + self.with_conn(sql_min_checkpoint_id) + } + + fn max_checkpoint_id(&self) -> Result, Self::Error> { + self.with_conn(sql_max_checkpoint_id) + } + + fn add_checkpoint( + &mut self, + checkpoint_id: Self::CheckpointId, + checkpoint: Checkpoint, + ) -> Result<(), Self::Error> { + self.with_conn(|conn| sql_add_checkpoint(conn, checkpoint_id, &checkpoint)) + } + + fn checkpoint_count(&self) -> Result { + self.with_conn(sql_checkpoint_count) + } + + fn get_checkpoint_at_depth( + &self, + checkpoint_depth: usize, + ) -> Result, Self::Error> { + self.with_conn(|conn| sql_get_checkpoint_at_depth(conn, checkpoint_depth)) + } + + fn get_checkpoint( + &self, + checkpoint_id: &Self::CheckpointId, + ) -> Result, Self::Error> { + self.with_conn(|conn| sql_get_checkpoint(conn, *checkpoint_id)) + } + + fn with_checkpoints(&mut self, limit: usize, mut callback: F) -> Result<(), Self::Error> + where + F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, + { + let entries = self.with_conn(|conn| sql_list_checkpoints(conn, limit))?; + for (id, checkpoint) in &entries { + callback(id, checkpoint)?; + } + Ok(()) + } + + fn for_each_checkpoint(&self, limit: usize, mut callback: F) -> Result<(), Self::Error> + where + F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, + { + let entries = self.with_conn(|conn| sql_list_checkpoints(conn, limit))?; + for (id, checkpoint) in &entries { + callback(id, checkpoint)?; + } + Ok(()) + } + + fn update_checkpoint_with( + &mut self, + checkpoint_id: &Self::CheckpointId, + update: F, + ) -> Result + where + F: Fn(&mut Checkpoint) -> Result<(), Self::Error>, + { + self.with_conn(|conn| sql_update_checkpoint_with(conn, *checkpoint_id, update)) + } + + fn remove_checkpoint(&mut self, checkpoint_id: &Self::CheckpointId) -> Result<(), Self::Error> { + self.with_conn(|conn| sql_remove_checkpoint(conn, *checkpoint_id)) + } + + fn truncate_checkpoints_retaining( + &mut self, + checkpoint_id: &Self::CheckpointId, + ) -> Result<(), Self::Error> { + self.with_conn(|conn| sql_truncate_checkpoints_retaining(conn, *checkpoint_id)) + } +} + +// --------------------------------------------------------------------------- +// Tree serialization +// --------------------------------------------------------------------------- + +/// Binary format tags for tree nodes. +const TAG_NIL: u8 = 0x00; +const TAG_LEAF: u8 = 0x01; +const TAG_PARENT: u8 = 0x02; + +/// Serialize a `PrunableTree` to bytes. +/// +/// Format: +/// - `Nil`: `[0x00]` +/// - `Leaf`: `[0x01][hash: 32][flags: 1]` +/// - `Parent`: `[0x02][has_ann: 1][ann?: 32][left][right]` +pub(crate) fn serialize_tree(tree: &PrunableTree) -> Vec { + let mut buf = Vec::new(); + serialize_tree_inner(tree, &mut buf); + buf +} + +fn serialize_tree_inner(tree: &PrunableTree, buf: &mut Vec) { + match &**tree { + Node::Nil => { + buf.push(TAG_NIL); + } + Node::Leaf { + value: (hash, flags), + } => { + buf.push(TAG_LEAF); + buf.extend_from_slice(&hash.to_bytes()); + buf.push(flags.bits()); + } + Node::Parent { ann, left, right } => { + buf.push(TAG_PARENT); + match ann { + Some(arc_hash) => { + buf.push(0x01); + buf.extend_from_slice(&arc_hash.to_bytes()); + } + None => { + buf.push(0x00); + } + } + serialize_tree_inner(left, buf); + serialize_tree_inner(right, buf); + } + } +} + +/// Deserialize a `PrunableTree` from bytes. +pub(crate) fn deserialize_tree( + data: &[u8], + pos: &mut usize, +) -> Result, SqliteShardStoreError> { + if *pos >= data.len() { + return Err(SqliteShardStoreError::Serialization( + "unexpected end of data".to_string(), + )); + } + + let tag = data[*pos]; + *pos += 1; + + match tag { + TAG_NIL => Ok(Tree::empty()), + TAG_LEAF => { + if *pos + 33 > data.len() { + return Err(SqliteShardStoreError::Serialization( + "truncated leaf data".to_string(), + )); + } + let hash_bytes: [u8; 32] = data[*pos..*pos + 32] + .try_into() + .map_err(|_| SqliteShardStoreError::Serialization("bad hash".to_string()))?; + *pos += 32; + let flags_byte = data[*pos]; + *pos += 1; + + let hash = merkle_hash_from_bytes(&hash_bytes).ok_or_else(|| { + SqliteShardStoreError::Serialization( + "invalid Pallas field element in leaf".to_string(), + ) + })?; + let flags = RetentionFlags::from_bits_truncate(flags_byte); + Ok(Tree::leaf((hash, flags))) + } + TAG_PARENT => { + if *pos >= data.len() { + return Err(SqliteShardStoreError::Serialization( + "truncated parent annotation flag".to_string(), + )); + } + let has_ann = data[*pos]; + *pos += 1; + + let ann: Option> = if has_ann == 0x01 { + if *pos + 32 > data.len() { + return Err(SqliteShardStoreError::Serialization( + "truncated parent annotation".to_string(), + )); + } + let ann_bytes: [u8; 32] = data[*pos..*pos + 32] + .try_into() + .map_err(|_| SqliteShardStoreError::Serialization("bad ann".to_string()))?; + *pos += 32; + let hash = merkle_hash_from_bytes(&ann_bytes).ok_or_else(|| { + SqliteShardStoreError::Serialization( + "invalid Pallas field element in annotation".to_string(), + ) + })?; + Some(Arc::new(hash)) + } else { + None + }; + + let left = deserialize_tree(data, pos)?; + let right = deserialize_tree(data, pos)?; + Ok(Tree::parent(ann, left, right)) + } + other => Err(SqliteShardStoreError::Serialization(format!( + "unknown tree node tag: 0x{other:02x}" + ))), + } +} diff --git a/grovedb-commitment-tree/src/client/sqlite_store_tests.rs b/grovedb-commitment-tree/src/client/sqlite_store_tests.rs new file mode 100644 index 000000000..b47016ea8 --- /dev/null +++ b/grovedb-commitment-tree/src/client/sqlite_store_tests.rs @@ -0,0 +1,444 @@ +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeSet, + sync::{Arc, Mutex}, + }; + + use incrementalmerkletree::{Address, Hashable, Level, Position}; + use orchard::tree::MerkleHashOrchard; + use rusqlite::Connection; + use shardtree::{ + store::{Checkpoint, ShardStore, TreeState}, + LocatedTree, Node, PrunableTree, RetentionFlags, Tree, + }; + + use crate::client::sqlite_store::{ + deserialize_tree, serialize_tree, SqliteShardStore, SHARD_HEIGHT, + }; + + fn test_store() -> SqliteShardStore { + let conn = Connection::open_in_memory().expect("open in-memory sqlite"); + SqliteShardStore::new(conn).expect("create store") + } + + fn test_hash(i: u8) -> MerkleHashOrchard { + let empty = MerkleHashOrchard::empty_leaf(); + MerkleHashOrchard::combine(Level::from(i % 31 + 1), &empty, &empty) + } + + // -- Schema tests -- + + #[test] + fn test_schema_creation() { + let store = test_store(); + let count: i64 = store + .with_conn(|conn| { + conn.query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name LIKE \ + 'commitment_tree_%'", + [], + |row| row.get(0), + ) + }) + .expect("query tables"); + assert_eq!(count, 4, "expected 4 commitment_tree_ tables"); + } + + #[test] + fn test_schema_idempotent() { + let conn = Connection::open_in_memory().expect("open in-memory sqlite"); + let _store = SqliteShardStore::new(conn).expect("first create"); + } + + #[test] + fn test_shared_connection() { + let conn = Connection::open_in_memory().expect("open in-memory sqlite"); + let arc = Arc::new(Mutex::new(conn)); + let mut store = SqliteShardStore::new_shared(arc.clone()).expect("create shared store"); + + // Store works + let addr = Address::from_parts(Level::from(SHARD_HEIGHT), 0); + let h = test_hash(1); + let tree = Tree::leaf((h, RetentionFlags::MARKED)); + let located = LocatedTree::from_parts(addr, tree).expect("create located"); + store.put_shard(located).expect("put shard via shared"); + + let retrieved = store.get_shard(addr).expect("get shard via shared"); + assert!(retrieved.is_some()); + + // Original Arc is still usable (mutex not poisoned) + let guard = arc.lock().expect("lock after store ops"); + let count: i64 = guard + .query_row("SELECT COUNT(*) FROM commitment_tree_shards", [], |row| { + row.get(0) + }) + .expect("direct query"); + assert_eq!(count, 1); + } + + // -- Serialization round-trip tests -- + + #[test] + fn test_serialize_nil() { + let tree: PrunableTree = Tree::empty(); + let data = serialize_tree(&tree); + let mut pos = 0; + let decoded = deserialize_tree(&data, &mut pos).expect("deserialize nil"); + assert!(decoded.is_empty()); + } + + #[test] + fn test_serialize_leaf() { + let hash = test_hash(1); + let flags = RetentionFlags::MARKED; + let tree = Tree::leaf((hash, flags)); + let data = serialize_tree(&tree); + assert_eq!(data.len(), 34); // 1 tag + 32 hash + 1 flags + let mut pos = 0; + let decoded = deserialize_tree(&data, &mut pos).expect("deserialize leaf"); + match &*decoded { + Node::Leaf { value: (h, f) } => { + assert_eq!(h.to_bytes(), hash.to_bytes()); + assert_eq!(*f, RetentionFlags::MARKED); + } + _ => panic!("expected leaf"), + } + } + + #[test] + fn test_serialize_parent_with_annotation() { + let h1 = test_hash(1); + let h2 = test_hash(2); + let h3 = test_hash(3); + let left = Tree::leaf((h1, RetentionFlags::MARKED)); + let right = Tree::leaf((h2, RetentionFlags::CHECKPOINT)); + let tree = Tree::parent(Some(Arc::new(h3)), left, right); + let data = serialize_tree(&tree); + let mut pos = 0; + let decoded = deserialize_tree(&data, &mut pos).expect("deserialize parent"); + match &*decoded { + Node::Parent { ann, .. } => { + assert!(ann.is_some()); + assert_eq!(ann.as_ref().expect("ann").to_bytes(), h3.to_bytes()); + } + _ => panic!("expected parent"), + } + } + + #[test] + fn test_serialize_parent_without_annotation() { + let h1 = test_hash(1); + let left = Tree::leaf((h1, RetentionFlags::EPHEMERAL)); + let right = Tree::empty(); + let tree: PrunableTree = Tree::parent(None, left, right); + let data = serialize_tree(&tree); + let mut pos = 0; + let decoded = deserialize_tree(&data, &mut pos).expect("deserialize parent no ann"); + match &*decoded { + Node::Parent { ann, .. } => { + assert!(ann.is_none()); + } + _ => panic!("expected parent"), + } + } + + #[test] + fn test_serialize_deep_tree() { + let h1 = test_hash(1); + let h2 = test_hash(2); + let h3 = test_hash(3); + let leaf1 = Tree::leaf((h1, RetentionFlags::MARKED)); + let leaf2 = Tree::leaf((h2, RetentionFlags::EPHEMERAL)); + let inner: PrunableTree = Tree::parent(None, leaf1, leaf2); + let leaf3 = Tree::leaf((h3, RetentionFlags::CHECKPOINT | RetentionFlags::MARKED)); + let root: PrunableTree = Tree::parent(Some(Arc::new(h1)), inner, leaf3); + let data = serialize_tree(&root); + let mut pos = 0; + let decoded = deserialize_tree(&data, &mut pos).expect("deserialize deep tree"); + assert_eq!(pos, data.len(), "should consume all bytes"); + match &*decoded { + Node::Parent { ann, left, right } => { + assert!(ann.is_some()); + match &***left { + Node::Parent { ann: inner_ann, .. } => { + let _: &Option> = inner_ann; + assert!(inner_ann.is_none()); + } + _ => panic!("expected inner parent"), + } + match &***right { + Node::Leaf { value: (_, f) } => { + assert!(f.is_checkpoint()); + assert!(f.is_marked()); + } + _ => panic!("expected leaf"), + } + } + _ => panic!("expected root parent"), + } + } + + // -- Shard CRUD tests -- + + #[test] + fn test_shard_round_trip() { + let mut store = test_store(); + let addr = Address::from_parts(Level::from(SHARD_HEIGHT), 0); + let h1 = test_hash(1); + let tree = Tree::leaf((h1, RetentionFlags::MARKED)); + let located = LocatedTree::from_parts(addr, tree).expect("create located tree"); + store.put_shard(located).expect("put shard"); + + let retrieved = store.get_shard(addr).expect("get shard"); + assert!(retrieved.is_some()); + let retrieved = retrieved.expect("shard should exist"); + assert_eq!(retrieved.root_addr(), addr); + } + + #[test] + fn test_shard_not_found() { + let store = test_store(); + let addr = Address::from_parts(Level::from(SHARD_HEIGHT), 42); + let result = store.get_shard(addr).expect("get shard"); + assert!(result.is_none()); + } + + #[test] + fn test_last_shard() { + let mut store = test_store(); + assert!(store.last_shard().expect("last shard empty").is_none()); + + for i in 0..3u64 { + let addr = Address::from_parts(Level::from(SHARD_HEIGHT), i); + let h = test_hash(i as u8); + let tree = Tree::leaf((h, RetentionFlags::EPHEMERAL)); + let located = LocatedTree::from_parts(addr, tree).expect("create located"); + store.put_shard(located).expect("put shard"); + } + + let last = store + .last_shard() + .expect("last shard") + .expect("should exist"); + assert_eq!(last.root_addr().index(), 2); + } + + #[test] + fn test_get_shard_roots() { + let mut store = test_store(); + assert!(store.get_shard_roots().expect("empty roots").is_empty()); + + for i in [0, 2, 5u64] { + let addr = Address::from_parts(Level::from(SHARD_HEIGHT), i); + let h = test_hash(i as u8); + let tree = Tree::leaf((h, RetentionFlags::EPHEMERAL)); + let located = LocatedTree::from_parts(addr, tree).expect("create located"); + store.put_shard(located).expect("put shard"); + } + + let roots = store.get_shard_roots().expect("roots"); + assert_eq!(roots.len(), 3); + assert_eq!(roots[0].index(), 0); + assert_eq!(roots[1].index(), 2); + assert_eq!(roots[2].index(), 5); + } + + #[test] + fn test_truncate_shards() { + let mut store = test_store(); + for i in 0..5u64 { + let addr = Address::from_parts(Level::from(SHARD_HEIGHT), i); + let h = test_hash(i as u8); + let tree = Tree::leaf((h, RetentionFlags::EPHEMERAL)); + let located = LocatedTree::from_parts(addr, tree).expect("create located"); + store.put_shard(located).expect("put shard"); + } + + store.truncate_shards(3).expect("truncate shards"); + let roots = store.get_shard_roots().expect("roots"); + assert_eq!(roots.len(), 3); + assert_eq!(roots.last().expect("last root").index(), 2); + } + + // -- Cap tests -- + + #[test] + fn test_cap_empty_default() { + let store = test_store(); + let cap = store.get_cap().expect("get cap"); + assert!(cap.is_empty()); + } + + #[test] + fn test_cap_round_trip() { + let mut store = test_store(); + let h = test_hash(10); + let cap: PrunableTree = Tree::leaf((h, RetentionFlags::EPHEMERAL)); + store.put_cap(cap).expect("put cap"); + let retrieved = store.get_cap().expect("get cap"); + match &*retrieved { + Node::Leaf { value: (hash, _) } => { + assert_eq!(hash.to_bytes(), h.to_bytes()); + } + _ => panic!("expected leaf cap"), + } + } + + // -- Checkpoint tests -- + + #[test] + fn test_checkpoint_empty() { + let store = test_store(); + assert_eq!(store.checkpoint_count().expect("count"), 0); + assert!(store.min_checkpoint_id().expect("min").is_none()); + assert!(store.max_checkpoint_id().expect("max").is_none()); + } + + #[test] + fn test_checkpoint_add_and_get() { + let mut store = test_store(); + let cp = Checkpoint::at_position(Position::from(42)); + store.add_checkpoint(1, cp).expect("add checkpoint"); + + assert_eq!(store.checkpoint_count().expect("count"), 1); + assert_eq!(store.min_checkpoint_id().expect("min"), Some(1)); + assert_eq!(store.max_checkpoint_id().expect("max"), Some(1)); + + let retrieved = store.get_checkpoint(&1).expect("get").expect("exists"); + assert_eq!( + retrieved.tree_state(), + TreeState::AtPosition(Position::from(42)) + ); + assert!(retrieved.marks_removed().is_empty()); + } + + #[test] + fn test_checkpoint_with_marks_removed() { + let mut store = test_store(); + let mut marks = BTreeSet::new(); + marks.insert(Position::from(10)); + marks.insert(Position::from(20)); + marks.insert(Position::from(30)); + let cp = Checkpoint::from_parts(TreeState::AtPosition(Position::from(50)), marks); + store.add_checkpoint(5, cp).expect("add checkpoint"); + + let retrieved = store.get_checkpoint(&5).expect("get").expect("exists"); + assert_eq!(retrieved.marks_removed().len(), 3); + assert!(retrieved.marks_removed().contains(&Position::from(10))); + assert!(retrieved.marks_removed().contains(&Position::from(20))); + assert!(retrieved.marks_removed().contains(&Position::from(30))); + } + + #[test] + fn test_checkpoint_at_depth() { + let mut store = test_store(); + for i in 1..=5u32 { + let cp = Checkpoint::at_position(Position::from(i as u64 * 10)); + store.add_checkpoint(i, cp).expect("add checkpoint"); + } + + let (id, cp) = store + .get_checkpoint_at_depth(0) + .expect("depth 0") + .expect("exists"); + assert_eq!(id, 5); + assert_eq!(cp.tree_state(), TreeState::AtPosition(Position::from(50))); + + let (id, _) = store + .get_checkpoint_at_depth(2) + .expect("depth 2") + .expect("exists"); + assert_eq!(id, 3); + + assert!(store + .get_checkpoint_at_depth(10) + .expect("depth 10") + .is_none()); + } + + #[test] + fn test_checkpoint_empty_tree_state() { + let mut store = test_store(); + let cp = Checkpoint::tree_empty(); + store.add_checkpoint(1, cp).expect("add empty checkpoint"); + + let retrieved = store.get_checkpoint(&1).expect("get").expect("exists"); + assert_eq!(retrieved.tree_state(), TreeState::Empty); + } + + #[test] + fn test_remove_checkpoint() { + let mut store = test_store(); + let mut marks = BTreeSet::new(); + marks.insert(Position::from(5)); + let cp = Checkpoint::from_parts(TreeState::AtPosition(Position::from(10)), marks); + store.add_checkpoint(1, cp).expect("add"); + store + .add_checkpoint(2, Checkpoint::tree_empty()) + .expect("add"); + + store.remove_checkpoint(&1).expect("remove"); + assert_eq!(store.checkpoint_count().expect("count"), 1); + assert!(store.get_checkpoint(&1).expect("get").is_none()); + assert!(store.get_checkpoint(&2).expect("get").is_some()); + } + + #[test] + fn test_truncate_checkpoints_retaining() { + let mut store = test_store(); + for i in 1..=5u32 { + let mut marks = BTreeSet::new(); + marks.insert(Position::from(i as u64)); + let cp = + Checkpoint::from_parts(TreeState::AtPosition(Position::from(i as u64 * 10)), marks); + store.add_checkpoint(i, cp).expect("add"); + } + + store.truncate_checkpoints_retaining(&3).expect("truncate"); + assert_eq!(store.checkpoint_count().expect("count"), 3); + assert_eq!(store.max_checkpoint_id().expect("max"), Some(3)); + + let cp3 = store.get_checkpoint(&3).expect("get").expect("exists"); + assert!(cp3.marks_removed().is_empty()); + + let cp2 = store.get_checkpoint(&2).expect("get").expect("exists"); + assert_eq!(cp2.marks_removed().len(), 1); + } + + #[test] + fn test_update_checkpoint_with() { + let mut store = test_store(); + let cp = Checkpoint::at_position(Position::from(10)); + store.add_checkpoint(1, cp).expect("add"); + + let updated = store + .update_checkpoint_with(&1, |_cp| Ok(())) + .expect("update"); + assert!(updated); + + let updated = store + .update_checkpoint_with(&999, |_| Ok(())) + .expect("update nonexistent"); + assert!(!updated); + } + + #[test] + fn test_for_each_checkpoint() { + let mut store = test_store(); + for i in 1..=5u32 { + store + .add_checkpoint(i, Checkpoint::at_position(Position::from(i as u64))) + .expect("add"); + } + + let mut ids = Vec::new(); + store + .for_each_checkpoint(3, |id, _| { + ids.push(*id); + Ok(()) + }) + .expect("for_each"); + assert_eq!(ids, vec![5, 4, 3]); + } +} diff --git a/grovedb-commitment-tree/src/client/tests.rs b/grovedb-commitment-tree/src/client/tests.rs new file mode 100644 index 000000000..db9ba5ef8 --- /dev/null +++ b/grovedb-commitment-tree/src/client/tests.rs @@ -0,0 +1,242 @@ +#[cfg(test)] +mod tests { + use incrementalmerkletree::{Hashable, Level, Position, Retention}; + use orchard::tree::{Anchor, MerkleHashOrchard}; + + use crate::ClientMemoryCommitmentTree; + + fn test_leaf(index: u64) -> [u8; 32] { + let empty = MerkleHashOrchard::empty_leaf(); + let varied = + MerkleHashOrchard::combine(Level::from((index % 31) as u8 + 1), &empty, &empty); + MerkleHashOrchard::combine(Level::from(0), &empty, &varied).to_bytes() + } + + #[test] + fn test_empty_tree() { + let tree = ClientMemoryCommitmentTree::new(10); + assert_eq!(tree.max_leaf_position().unwrap(), None); + assert_eq!(tree.anchor().unwrap(), Anchor::empty_tree()); + } + + #[test] + fn test_append_and_position() { + let mut tree = ClientMemoryCommitmentTree::new(10); + + tree.append(test_leaf(0), Retention::Marked).unwrap(); + assert_eq!(tree.max_leaf_position().unwrap(), Some(Position::from(0))); + + tree.append(test_leaf(1), Retention::Ephemeral).unwrap(); + assert_eq!(tree.max_leaf_position().unwrap(), Some(Position::from(1))); + } + + #[test] + fn test_anchor_changes() { + let mut tree = ClientMemoryCommitmentTree::new(10); + let empty_anchor = tree.anchor().unwrap(); + + tree.append(test_leaf(0), Retention::Marked).unwrap(); + let anchor1 = tree.anchor().unwrap(); + assert_ne!(empty_anchor, anchor1); + + tree.append(test_leaf(1), Retention::Marked).unwrap(); + let anchor2 = tree.anchor().unwrap(); + assert_ne!(anchor1, anchor2); + } + + #[test] + fn test_witness_generation() { + let mut tree = ClientMemoryCommitmentTree::new(10); + + // Append a marked leaf so we can witness it + tree.append(test_leaf(0), Retention::Marked).unwrap(); + tree.append(test_leaf(1), Retention::Ephemeral).unwrap(); + tree.checkpoint(1).unwrap(); + + // Witness for position 0 at current state + let path = tree.witness(Position::from(0), 0).unwrap(); + assert!(path.is_some(), "should produce witness for marked leaf"); + } + + #[test] + #[cfg(feature = "server")] + fn test_frontier_and_client_same_root() { + use crate::commitment_frontier::CommitmentFrontier; + + let mut frontier = CommitmentFrontier::new(); + let mut client = ClientMemoryCommitmentTree::new(10); + + for i in 0..20u64 { + frontier + .append(test_leaf(i)) + .value + .expect("frontier append"); + client.append(test_leaf(i), Retention::Ephemeral).unwrap(); + } + + assert_eq!(frontier.anchor(), client.anchor().unwrap()); + } + + /// Demonstrates that `checkpoint()` with a duplicate ID silently returns + /// `Ok(false)` and does NOT advance the checkpoint frontier. Notes + /// appended after the original checkpoint are unreachable by + /// `witness_at_checkpoint_depth(pos, 0)`. + /// + /// This is the exact failure mode that caused the "Tree does not contain + /// a root at address" error in PMT when the sync code reused + /// `next_start_index` as the checkpoint ID across re-syncs. + #[test] + fn test_duplicate_checkpoint_id_breaks_witness_for_new_notes() { + let mut tree = ClientMemoryCommitmentTree::new(100); + + // Sync 1: append 20 notes (even = Marked, odd = Ephemeral) + for i in 0..20u64 { + let retention = if i % 2 == 0 { + Retention::Marked + } else { + Retention::Ephemeral + }; + tree.append(test_leaf(i), retention).expect("append sync 1"); + } + + // Checkpoint with the "chunk boundary" ID + let created = tree.checkpoint(2048).expect("checkpoint 1"); + assert!(created, "first checkpoint should succeed"); + + // Witness works for all marked notes in sync 1 + for i in (0..20u64).step_by(2) { + let path = tree + .witness(Position::from(i), 0) + .expect("witness sync 1 note"); + assert!( + path.is_some(), + "should produce witness for marked note at position {}", + i + ); + } + + // Sync 2: append 30 more notes (simulates new notes arriving) + for i in 20..50u64 { + let retention = if i % 2 == 0 { + Retention::Marked + } else { + Retention::Ephemeral + }; + tree.append(test_leaf(i), retention).expect("append sync 2"); + } + + // BUG: reuse the same checkpoint ID — returns Ok(false)! + let created = tree.checkpoint(2048).expect("checkpoint 2 (duplicate)"); + assert!( + !created, + "duplicate checkpoint ID should return false (no new checkpoint created)" + ); + + // Original sync 1 notes still have valid witnesses + let path = tree + .witness(Position::from(0), 0) + .expect("witness sync 1 note after sync 2"); + assert!(path.is_some(), "sync 1 notes should still be witnessable"); + + // Sync 2 notes at positions >= 20 CANNOT be witnessed because the + // checkpoint is stuck at position 19 (from sync 1). This is the bug. + let result = tree.witness(Position::from(20), 0); + assert!( + result.is_err(), + "witness should fail for notes beyond the stale checkpoint" + ); + } + + /// Shows the correct pattern: use unique, increasing checkpoint IDs + /// so that each sync creates a new checkpoint covering all appended notes. + #[test] + fn test_unique_checkpoint_ids_allow_witness_for_all_notes() { + let mut tree = ClientMemoryCommitmentTree::new(100); + + // Sync 1: append 20 notes + for i in 0..20u64 { + let retention = if i % 2 == 0 { + Retention::Marked + } else { + Retention::Ephemeral + }; + tree.append(test_leaf(i), retention).expect("append sync 1"); + } + + // Checkpoint with unique ID = last appended position + let created = tree.checkpoint(19).expect("checkpoint 1"); + assert!(created, "first checkpoint should succeed"); + + // Sync 2: append 30 more notes + for i in 20..50u64 { + let retention = if i % 2 == 0 { + Retention::Marked + } else { + Retention::Ephemeral + }; + tree.append(test_leaf(i), retention).expect("append sync 2"); + } + + // Checkpoint with new unique ID = new last appended position + let created = tree.checkpoint(49).expect("checkpoint 2"); + assert!(created, "second checkpoint with unique ID should succeed"); + + // ALL marked notes — from both syncs — can be witnessed + for i in (0..50u64).step_by(2) { + let path = tree + .witness(Position::from(i), 0) + .expect(&format!("witness note at position {}", i)); + assert!( + path.is_some(), + "should produce witness for marked note at position {}", + i + ); + } + } + + /// Verifies that witness anchors from both syncs match when using + /// unique checkpoint IDs, and that the anchor at checkpoint depth 1 + /// differs from depth 0 (since the tree grew between checkpoints). + #[test] + fn test_witness_anchors_match_across_syncs() { + let mut tree = ClientMemoryCommitmentTree::new(100); + + // Sync 1 + for i in 0..10u64 { + tree.append(test_leaf(i), Retention::Marked) + .expect("append sync 1"); + } + tree.checkpoint(9).expect("checkpoint 1"); + let anchor_after_sync1 = tree.anchor().expect("anchor after sync 1"); + + // Sync 2 + for i in 10..20u64 { + tree.append(test_leaf(i), Retention::Marked) + .expect("append sync 2"); + } + tree.checkpoint(19).expect("checkpoint 2"); + let anchor_after_sync2 = tree.anchor().expect("anchor after sync 2"); + + // Anchors should differ (tree grew) + assert_ne!( + anchor_after_sync1, anchor_after_sync2, + "anchors should differ after tree growth" + ); + + // Witness at depth 0 uses the latest checkpoint (sync 2) + let path_depth0 = tree + .witness(Position::from(0), 0) + .expect("witness at depth 0"); + assert!(path_depth0.is_some()); + + // Witness at depth 1 uses the previous checkpoint (sync 1) + let path_depth1 = tree + .witness(Position::from(0), 1) + .expect("witness at depth 1"); + assert!(path_depth1.is_some()); + + // Both witnesses exist at their respective checkpoint depths + // (MerklePath doesn't implement PartialEq so we just verify both are + // Some) + } +} diff --git a/grovedb-commitment-tree/src/commitment_frontier/mod.rs b/grovedb-commitment-tree/src/commitment_frontier/mod.rs new file mode 100644 index 000000000..36f461e87 --- /dev/null +++ b/grovedb-commitment-tree/src/commitment_frontier/mod.rs @@ -0,0 +1,230 @@ +use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use incrementalmerkletree::{frontier::Frontier, Hashable, Level, Position}; +use orchard::{tree::MerkleHashOrchard, Anchor, NOTE_COMMITMENT_TREE_DEPTH}; + +pub use crate::error::CommitmentTreeError; + +mod tests; + +/// Depth of the Sinsemilla Merkle tree as a u8 constant for the Frontier type +/// parameter. +#[cfg(feature = "server")] +const FRONTIER_DEPTH: u8 = NOTE_COMMITMENT_TREE_DEPTH as u8; + +/// A lightweight frontier-based Sinsemilla commitment tree. +/// +/// Stores only the rightmost path of the depth-32 Merkle tree (~1KB), +/// supporting O(1) append and root hash computation. +/// +/// The full note data (cmx || encrypted_note) is stored separately as +/// items in a GroveDB CountTree. This struct only tracks the Sinsemilla +/// hash state. Historical anchors for spend authorization are managed +/// by Platform in a separate provable tree. +/// +/// Requires the `server` feature. +#[cfg(feature = "server")] +#[derive(Debug, Clone)] +pub struct CommitmentFrontier { + frontier: Frontier, +} + +#[cfg(feature = "server")] +impl CommitmentFrontier { + /// Create a new empty commitment frontier. + pub fn new() -> Self { + Self { + frontier: Frontier::empty(), + } + } + + /// Append a commitment (cmx) to the frontier. + /// + /// Returns the new Sinsemilla root hash after the append. The returned + /// [`OperationCost`] tracks `sinsemilla_hash_calls`: 32 hashes for the + /// leaf-to-root path plus `trailing_ones(position)` ommer hashes. + pub fn append(&mut self, cmx: [u8; 32]) -> CostResult<[u8; 32], CommitmentTreeError> { + let mut cost = OperationCost::default(); + let leaf = match merkle_hash_from_bytes(&cmx) { + Some(l) => l, + None => { + return Err(CommitmentTreeError::InvalidFieldElement).wrap_with_cost(cost); + } + }; + + // Count Sinsemilla hashes: 32 levels for the leaf path + trailing_ones + // for ommer merges + let ommer_hashes = self + .frontier + .value() + .map(|f| u64::from(f.position()).trailing_ones()) + .unwrap_or(0); + cost.sinsemilla_hash_calls += 32 + ommer_hashes; + + if !self.frontier.append(leaf) { + return Err(CommitmentTreeError::TreeFull).wrap_with_cost(cost); + } + Ok(self.root_hash()).wrap_with_cost(cost) + } + + /// Get the current Sinsemilla root hash as 32 bytes. + /// + /// Returns the empty tree root if no leaves have been appended. + pub fn root_hash(&self) -> [u8; 32] { + self.frontier.root().to_bytes() + } + + /// Get the current root as an Orchard `Anchor`. + pub fn anchor(&self) -> Anchor { + Anchor::from(self.frontier.root()) + } + + /// Get the position of the most recently appended leaf. + /// + /// Returns `None` if the frontier is empty. The position is 0-indexed, + /// so it equals `count - 1`. + pub fn position(&self) -> Option { + self.frontier.value().map(|f| u64::from(f.position())) + } + + /// Get the number of leaves that have been appended. + pub fn tree_size(&self) -> u64 { + self.frontier.tree_size() + } + + /// Serialize the frontier to bytes. + /// + /// Format: + /// ```text + /// has_frontier: u8 (0x00 = empty, 0x01 = non-empty) + /// If non-empty: + /// position: u64 BE (8 bytes) + /// leaf: [u8; 32] + /// ommer_count: u8 + /// ommers: [ommer_count × 32 bytes] + /// ``` + pub fn serialize(&self) -> Vec { + let mut buf = Vec::new(); + + match self.frontier.value() { + None => { + buf.push(0x00); + } + Some(f) => { + buf.push(0x01); + buf.extend_from_slice(&u64::from(f.position()).to_be_bytes()); + buf.extend_from_slice(&f.leaf().to_bytes()); + let ommers = f.ommers(); + buf.push(ommers.len() as u8); + for ommer in ommers { + buf.extend_from_slice(&ommer.to_bytes()); + } + } + } + + buf + } + + /// Deserialize a frontier from bytes. + pub fn deserialize(data: &[u8]) -> Result { + if data.is_empty() { + return Err(CommitmentTreeError::InvalidData("empty input".to_string())); + } + + let mut pos = 0; + + let has_frontier = data[pos]; + pos += 1; + + let frontier = if has_frontier == 0x00 { + Frontier::empty() + } else if has_frontier == 0x01 { + if data.len() < pos + 8 + 32 + 1 { + return Err(CommitmentTreeError::InvalidData( + "truncated frontier header".to_string(), + )); + } + + let position_u64 = u64::from_be_bytes( + data[pos..pos + 8] + .try_into() + .map_err(|_| CommitmentTreeError::InvalidData("bad position".to_string()))?, + ); + pos += 8; + + let leaf_bytes: [u8; 32] = data[pos..pos + 32] + .try_into() + .map_err(|_| CommitmentTreeError::InvalidData("bad leaf".to_string()))?; + let leaf = merkle_hash_from_bytes(&leaf_bytes) + .ok_or(CommitmentTreeError::InvalidFieldElement)?; + pos += 32; + + let ommer_count = data[pos] as usize; + pos += 1; + + if data.len() < pos + ommer_count * 32 { + return Err(CommitmentTreeError::InvalidData( + "truncated ommers".to_string(), + )); + } + + let mut ommers = Vec::with_capacity(ommer_count); + for _ in 0..ommer_count { + let ommer_bytes: [u8; 32] = data[pos..pos + 32] + .try_into() + .map_err(|_| CommitmentTreeError::InvalidData("bad ommer".to_string()))?; + let ommer = merkle_hash_from_bytes(&ommer_bytes) + .ok_or(CommitmentTreeError::InvalidFieldElement)?; + ommers.push(ommer); + pos += 32; + } + + // Allow trailing bytes for forward compatibility (old serialization + // included historical anchors after the frontier data). + let _ = pos; + + Frontier::from_parts(Position::from(position_u64), leaf, ommers).map_err(|e| { + CommitmentTreeError::InvalidData(format!("frontier reconstruction: {:?}", e)) + })? + } else { + return Err(CommitmentTreeError::InvalidData(format!( + "invalid frontier flag: 0x{:02x}", + has_frontier + ))); + }; + + Ok(Self { frontier }) + } +} + +#[cfg(feature = "server")] +impl Default for CommitmentFrontier { + fn default() -> Self { + Self::new() + } +} + +/// Convert raw 32 bytes to a `MerkleHashOrchard`, returning `None` if the +/// bytes do not represent a valid Pallas field element. +pub fn merkle_hash_from_bytes(bytes: &[u8; 32]) -> Option { + Option::from(MerkleHashOrchard::from_bytes(bytes)) +} + +/// Return the Sinsemilla root hash of an empty depth-32 commitment tree. +/// +/// This is the root when zero leaves have been appended. It equals +/// `MerkleHashOrchard::empty_root(Level::from(32))`. +/// +/// The value is computed once and cached. It is also available as the +/// constant [`EMPTY_SINSEMILLA_ROOT`]. +pub fn empty_sinsemilla_root() -> [u8; 32] { + MerkleHashOrchard::empty_root(Level::from(NOTE_COMMITMENT_TREE_DEPTH as u8)).to_bytes() +} + +/// Precomputed Sinsemilla root of an empty depth-32 commitment tree. +/// +/// Generated by `MerkleHashOrchard::empty_root(Level::from(32)).to_bytes()`. +/// Verified at compile time via `grovedb-commitment-tree` unit tests. +pub const EMPTY_SINSEMILLA_ROOT: [u8; 32] = [ + 0xae, 0x29, 0x35, 0xf1, 0xdf, 0xd8, 0xa2, 0x4a, 0xed, 0x7c, 0x70, 0xdf, 0x7d, 0xe3, 0xa6, 0x68, + 0xeb, 0x7a, 0x49, 0xb1, 0x31, 0x98, 0x80, 0xdd, 0xe2, 0xbb, 0xd9, 0x03, 0x1a, 0xe5, 0xd8, 0x2f, +]; diff --git a/grovedb-commitment-tree/src/commitment_frontier/tests.rs b/grovedb-commitment-tree/src/commitment_frontier/tests.rs new file mode 100644 index 000000000..3febace53 --- /dev/null +++ b/grovedb-commitment-tree/src/commitment_frontier/tests.rs @@ -0,0 +1,318 @@ +#[cfg(all(test, feature = "server"))] +mod tests { + use incrementalmerkletree::{Hashable, Level}; + use orchard::{ + tree::{Anchor, MerkleHashOrchard}, + NOTE_COMMITMENT_TREE_DEPTH, + }; + + use crate::commitment_frontier::{ + empty_sinsemilla_root, CommitmentFrontier, EMPTY_SINSEMILLA_ROOT, + }; + + /// Create a deterministic test leaf from an index. + fn test_leaf(index: u64) -> [u8; 32] { + let empty = MerkleHashOrchard::empty_leaf(); + let varied = + MerkleHashOrchard::combine(Level::from((index % 31) as u8 + 1), &empty, &empty); + MerkleHashOrchard::combine(Level::from(0), &empty, &varied).to_bytes() + } + + #[test] + fn test_empty_frontier() { + let f = CommitmentFrontier::new(); + assert_eq!(f.position(), None); + assert_eq!(f.tree_size(), 0); + + let empty_anchor = Anchor::empty_tree(); + assert_eq!(f.anchor(), empty_anchor); + } + + #[test] + fn test_append_changes_root() { + let mut f = CommitmentFrontier::new(); + let empty_root = f.root_hash(); + + let result = f.append(test_leaf(0)); + let new_root = result.value.expect("append should succeed"); + assert_ne!(empty_root, new_root); + assert_eq!(f.root_hash(), new_root); + } + + #[test] + fn test_append_tracks_position() { + let mut f = CommitmentFrontier::new(); + assert_eq!(f.position(), None); + assert_eq!(f.tree_size(), 0); + + f.append(test_leaf(0)).value.expect("append 0"); + assert_eq!(f.position(), Some(0)); + assert_eq!(f.tree_size(), 1); + + f.append(test_leaf(1)).value.expect("append 1"); + assert_eq!(f.position(), Some(1)); + assert_eq!(f.tree_size(), 2); + + for i in 2..100u64 { + f.append(test_leaf(i)).value.expect("append loop"); + } + assert_eq!(f.position(), Some(99)); + assert_eq!(f.tree_size(), 100); + } + + #[test] + fn test_deterministic_roots() { + let mut f1 = CommitmentFrontier::new(); + let mut f2 = CommitmentFrontier::new(); + + for i in 0..10u64 { + f1.append(test_leaf(i)).value.expect("append f1"); + f2.append(test_leaf(i)).value.expect("append f2"); + } + + assert_eq!(f1.root_hash(), f2.root_hash()); + } + + #[test] + fn test_different_leaves_different_roots() { + let mut f1 = CommitmentFrontier::new(); + let mut f2 = CommitmentFrontier::new(); + + f1.append(test_leaf(0)).value.expect("append f1"); + f2.append(test_leaf(1)).value.expect("append f2"); + + assert_ne!(f1.root_hash(), f2.root_hash()); + } + + #[test] + fn test_serialize_empty() { + let f = CommitmentFrontier::new(); + let data = f.serialize(); + let f2 = CommitmentFrontier::deserialize(&data).unwrap(); + + assert_eq!(f.root_hash(), f2.root_hash()); + assert_eq!(f.position(), f2.position()); + } + + #[test] + fn test_serialize_roundtrip() { + let mut f = CommitmentFrontier::new(); + for i in 0..100u64 { + f.append(test_leaf(i)).value.expect("append"); + } + + let data = f.serialize(); + let f2 = CommitmentFrontier::deserialize(&data).unwrap(); + + assert_eq!(f.root_hash(), f2.root_hash()); + assert_eq!(f.position(), f2.position()); + assert_eq!(f.tree_size(), f2.tree_size()); + } + + #[test] + fn test_serialize_roundtrip_with_many_leaves() { + let mut f = CommitmentFrontier::new(); + for i in 0..1000u64 { + f.append(test_leaf(i)).value.expect("append"); + } + + let data = f.serialize(); + // Frontier should be small regardless of leaf count + // 1 (flag) + 8 (position) + 32 (leaf) + 1 (ommer_count) + N*32 (ommers) + // Max ommers for depth 32 = 32, so max ~1.1KB + assert!( + data.len() < 1200, + "frontier serialized to {} bytes", + data.len() + ); + + let f2 = CommitmentFrontier::deserialize(&data).unwrap(); + assert_eq!(f.root_hash(), f2.root_hash()); + assert_eq!(f.tree_size(), f2.tree_size()); + } + + #[test] + fn test_invalid_field_element() { + // All 0xFF bytes is not a valid Pallas field element + let result = CommitmentFrontier::new().append([0xff; 32]); + assert!(result.value.is_err()); + } + + #[test] + fn test_deserialize_invalid_data() { + assert!(CommitmentFrontier::deserialize(&[]).is_err()); + assert!(CommitmentFrontier::deserialize(&[0x02]).is_err()); + assert!(CommitmentFrontier::deserialize(&[0x01]).is_err()); + } + + #[test] + fn test_root_hash_is_32_bytes() { + let f = CommitmentFrontier::new(); + assert_eq!(f.root_hash().len(), 32); + } + + #[test] + fn test_empty_tree_root_matches_orchard() { + let f = CommitmentFrontier::new(); + let root = f.root_hash(); + let expected = + MerkleHashOrchard::empty_root(Level::from(NOTE_COMMITMENT_TREE_DEPTH as u8)).to_bytes(); + assert_eq!(root, expected); + } + + #[test] + fn test_empty_sinsemilla_root_constant() { + // Verify the precomputed constant matches the runtime value + let computed = empty_sinsemilla_root(); + assert_eq!( + computed, EMPTY_SINSEMILLA_ROOT, + "EMPTY_SINSEMILLA_ROOT constant is stale. Update it to: {:?}", + computed + ); + } + + #[test] + fn test_default_impl() { + let f = CommitmentFrontier::default(); + assert_eq!(f.position(), None); + assert_eq!(f.tree_size(), 0); + assert_eq!(f.root_hash(), CommitmentFrontier::new().root_hash()); + } + + #[test] + fn test_deserialize_truncated_ommers() { + // Build a valid serialized frontier with 1 leaf so we know the ommer + // count byte, then truncate the ommer data. + let mut f = CommitmentFrontier::new(); + // Append enough leaves to generate ommers. After 3 appends (positions + // 0,1,2), position=2 has trailing_ones=0 so ommer_count may be 1. + // After 4 appends position=3 has trailing_ones=2, generating ommers. + for i in 0..4u64 { + f.append(test_leaf(i)).value.expect("append"); + } + let data = f.serialize(); + // data layout: 1 (flag) + 8 (position) + 32 (leaf) + 1 (ommer_count) + N*32 + let ommer_count = data[42] as usize; + assert!( + ommer_count > 0, + "need at least one ommer to test truncation" + ); + // Truncate: keep header + ommer_count byte but chop the ommer data + let truncated = &data[..43]; + let err = CommitmentFrontier::deserialize(truncated); + assert!(err.is_err(), "should fail on truncated ommers"); + let msg = format!("{}", err.unwrap_err()); + assert!( + msg.contains("truncated ommers"), + "expected 'truncated ommers' error, got: {msg}" + ); + } + + #[test] + fn test_deserialize_invalid_leaf_field_element() { + // Construct bytes with valid header but an invalid Pallas field element + // as the leaf (all 0xFF is not a valid point). + let mut data = vec![0x01]; // has_frontier = true + data.extend_from_slice(&0u64.to_be_bytes()); // position = 0 + data.extend_from_slice(&[0xFF; 32]); // invalid leaf + data.push(0); // ommer_count = 0 + + let err = CommitmentFrontier::deserialize(&data); + assert!(err.is_err(), "should fail on invalid leaf field element"); + let msg = format!("{}", err.unwrap_err()); + assert!( + msg.contains("invalid Pallas field element"), + "expected InvalidFieldElement error, got: {msg}" + ); + } + + #[test] + fn test_deserialize_invalid_ommer_field_element() { + // Build a valid frontier, then replace one ommer with 0xFF bytes. + let mut f = CommitmentFrontier::new(); + for i in 0..4u64 { + f.append(test_leaf(i)).value.expect("append"); + } + let mut data = f.serialize(); + let ommer_count = data[42] as usize; + assert!(ommer_count > 0, "need at least one ommer"); + // First ommer starts at byte 43, replace it with all 0xFF + for b in &mut data[43..43 + 32] { + *b = 0xFF; + } + let err = CommitmentFrontier::deserialize(&data); + assert!(err.is_err(), "should fail on invalid ommer field element"); + let msg = format!("{}", err.unwrap_err()); + assert!( + msg.contains("invalid Pallas field element"), + "expected InvalidFieldElement error, got: {msg}" + ); + } + + #[test] + fn test_deserialize_from_parts_failure() { + // Construct technically valid field elements but with an inconsistent + // position/ommer combination that `Frontier::from_parts` rejects. + // Position 0 should have 0 ommers; providing 1 ommer triggers the + // from_parts validation error. + let valid_leaf = test_leaf(0); + let valid_ommer = test_leaf(1); + + let mut data = vec![0x01]; // has_frontier + data.extend_from_slice(&0u64.to_be_bytes()); // position = 0 + data.extend_from_slice(&valid_leaf); // leaf + data.push(1); // ommer_count = 1 (wrong for position 0) + data.extend_from_slice(&valid_ommer); // ommer + + let err = CommitmentFrontier::deserialize(&data); + assert!(err.is_err(), "should fail on inconsistent from_parts"); + let msg = format!("{}", err.unwrap_err()); + assert!( + msg.contains("frontier reconstruction"), + "expected 'frontier reconstruction' error, got: {msg}" + ); + } + + #[test] + fn test_append_cost_sinsemilla_hash_calls() { + let mut f = CommitmentFrontier::new(); + + // First append (position 0): 32 hashes + 0 trailing_ones(empty) = 32 + let r0 = f.append(test_leaf(0)); + r0.value.expect("append 0"); + assert_eq!(r0.cost.sinsemilla_hash_calls, 32); + + // Second append (position 0 in frontier before append): trailing_ones(0) + // = 0 0 in binary is ...0, trailing_ones = 0, so 32 + 0 = 32 + let r1 = f.append(test_leaf(1)); + r1.value.expect("append 1"); + assert_eq!(r1.cost.sinsemilla_hash_calls, 32); + + // Third append (position 1): trailing_ones(1) = 1, so 32 + 1 = 33 + let r2 = f.append(test_leaf(2)); + r2.value.expect("append 2"); + assert_eq!(r2.cost.sinsemilla_hash_calls, 33); + + // Fourth append (position 2): trailing_ones(2=0b10) = 0, so 32 + let r3 = f.append(test_leaf(3)); + r3.value.expect("append 3"); + assert_eq!(r3.cost.sinsemilla_hash_calls, 32); + + // Fifth append (position 3): trailing_ones(3=0b11) = 2, so 34 + let r4 = f.append(test_leaf(4)); + r4.value.expect("append 4"); + assert_eq!(r4.cost.sinsemilla_hash_calls, 34); + } + + #[test] + fn test_deserialize_invalid_frontier_flag() { + // Test with a frontier flag value that is neither 0x00 nor 0x01 + let err = CommitmentFrontier::deserialize(&[0x42]); + assert!(err.is_err()); + let msg = format!("{}", err.unwrap_err()); + assert!( + msg.contains("invalid frontier flag: 0x42"), + "expected 'invalid frontier flag' error, got: {msg}" + ); + } +} diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs new file mode 100644 index 000000000..a6379c2e4 --- /dev/null +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -0,0 +1,353 @@ +//! Storage adapter bridging GroveDB's `StorageContext` to the composite +//! commitment tree. +//! +//! Provides [`CommitmentTree`], which owns both the in-memory +//! [`CommitmentFrontier`] and a [`BulkAppendTree`], combining the Sinsemilla +//! frontier (for anchor computation) with the two-level append-only store (for +//! `cmx||payload` persistence with epoch compaction) into a single struct. +//! +//! All mutating operations return [`CostResult`] to propagate storage costs. + +use std::marker::PhantomData; + +use grovedb_bulk_append_tree::BulkAppendTree; +use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use grovedb_storage::StorageContext; +use orchard::{ + memo::{DashMemo, MemoSize}, + note::TransmittedNoteCiphertext, + zcash_note_encryption::note_bytes::NoteBytes, +}; + +use crate::{CommitmentFrontier, CommitmentTreeError}; + +mod tests; + +/// Key used to store the serialized commitment frontier in data storage. +pub const COMMITMENT_TREE_DATA_KEY: &[u8] = b"__ct_data__"; + +/// Result of appending to a [`CommitmentTree`]. +#[derive(Debug, Clone)] +pub struct CommitmentAppendResult { + /// The new Sinsemilla frontier root hash. + pub sinsemilla_root: [u8; 32], + /// The BulkAppendTree state root (`blake3(mmr_root || dense_tree_root)`). + /// This flows as the Merk child hash via `insert_subtree`. + pub bulk_state_root: [u8; 32], + /// The 0-based global position of the appended value. + pub global_position: u64, + /// Number of blake3 hash calls performed during the bulk append. + pub hash_count: u32, + /// Whether compaction (epoch flush) occurred during this append. + pub compacted: bool, +} + +// ── Ciphertext serialization helpers ───────────────────────────────────── + +/// Compute the expected ciphertext payload size (excluding the 32-byte cmx +/// prefix) for a given `MemoSize`. +/// +/// Layout: `epk_bytes (32) || enc_ciphertext (variable) || out_ciphertext (80)` +/// +/// For `DashMemo`: `32 + 104 + 80 = 216 bytes`. +pub fn ciphertext_payload_size() -> usize { + 32 + std::mem::size_of::() + 80 +} + +/// Serialize a [`TransmittedNoteCiphertext`] to bytes. +/// +/// Output layout: `epk_bytes (32) || enc_ciphertext || out_ciphertext (80)` +pub fn serialize_ciphertext(ct: &TransmittedNoteCiphertext) -> Vec { + let enc = ct.enc_ciphertext.as_ref(); + let mut buf = Vec::with_capacity(32 + enc.len() + 80); + buf.extend_from_slice(&ct.epk_bytes); + buf.extend_from_slice(enc); + buf.extend_from_slice(&ct.out_ciphertext); + buf +} + +/// Deserialize a [`TransmittedNoteCiphertext`] from bytes. +/// +/// Expected layout: `epk_bytes (32) || enc_ciphertext || out_ciphertext (80)` +pub fn deserialize_ciphertext(data: &[u8]) -> Option> { + let enc_size = data.len().checked_sub(32 + 80)?; + let epk_bytes: [u8; 32] = data[..32].try_into().ok()?; + let enc_ciphertext = + ::from_slice(&data[32..32 + enc_size])?; + let out_ciphertext: [u8; 80] = data[32 + enc_size..].try_into().ok()?; + Some(TransmittedNoteCiphertext::from_parts( + epk_bytes, + enc_ciphertext, + out_ciphertext, + )) +} + +/// Commitment tree combining in-memory frontier state with a +/// [`BulkAppendTree`]. +/// +/// Owns both the [`CommitmentFrontier`] (Sinsemilla anchor computation) and a +/// [`BulkAppendTree`] (efficient append-only storage with epoch compaction). +/// Storage is owned by the `BulkAppendTree` via its dense tree. +/// +/// The type parameter `M` controls the memo size for note ciphertext +/// validation. It defaults to [`DashMemo`] so code that doesn't care about M +/// (like `verify_grovedb`, `commitment_tree_anchor`) works without specifying +/// it. +/// +/// - [`open`](CommitmentTree::open) loads the frontier from storage (or starts +/// empty) and reconstructs the `BulkAppendTree` from persisted state +/// - [`append`](CommitmentTree::append) appends `cmx||ciphertext` to the bulk +/// tree and `cmx` to the frontier +/// - [`save`](CommitmentTree::save) persists the frontier back to storage +pub struct CommitmentTree { + frontier: CommitmentFrontier, + pub bulk_tree: BulkAppendTree, + _memo: PhantomData, +} + +impl std::fmt::Debug for CommitmentTree { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CommitmentTree") + .field("frontier", &self.frontier) + .field("total_count", &self.bulk_tree.total_count) + .field("memo_type", &std::any::type_name::()) + .finish_non_exhaustive() + } +} + +impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { + /// Create a new empty commitment tree. + /// + /// `chunk_power` is the log2 of the epoch size for the underlying + /// `BulkAppendTree` (height parameter). + pub fn new(chunk_power: u8, storage: S) -> Result { + let bulk_tree = BulkAppendTree::new(chunk_power, storage) + .map_err(|e| CommitmentTreeError::InvalidData(format!("bulk tree new: {}", e)))?; + Ok(Self { + frontier: CommitmentFrontier::new(), + bulk_tree, + _memo: PhantomData, + }) + } + + /// Load a commitment tree from storage, or start with an empty frontier if + /// no data exists yet. + /// + /// Reconstructs the `BulkAppendTree` from `total_count` and `chunk_power`, + /// then reads the serialized `CommitmentFrontier` from storage. + pub fn open( + total_count: u64, + chunk_power: u8, + storage: S, + ) -> CostResult { + let mut cost = OperationCost::default(); + + let bulk_tree = match BulkAppendTree::from_state(total_count, chunk_power, storage) { + Ok(t) => t, + Err(e) => { + return Err(CommitmentTreeError::InvalidData(format!( + "bulk tree from_state: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + + // Read frontier from the bulk tree's storage + let data = bulk_tree + .dense_tree + .storage + .get(COMMITMENT_TREE_DATA_KEY) + .unwrap_add_cost(&mut cost); + + let frontier = match data { + Ok(Some(bytes)) => match CommitmentFrontier::deserialize(&bytes) { + Ok(f) => f, + Err(e) => return Err(e).wrap_with_cost(cost), + }, + Ok(None) => CommitmentFrontier::new(), + Err(e) => { + return Err(CommitmentTreeError::InvalidData(format!( + "storage error loading frontier: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + + Ok(Self { + frontier, + bulk_tree, + _memo: PhantomData, + }) + .wrap_with_cost(cost) + } + + /// Append a typed ciphertext and note commitment to the commitment tree. + /// + /// This is the primary typed API. It serializes the ciphertext internally + /// and delegates to [`append_raw`](Self::append_raw). + /// + /// Call [`save`](Self::save) afterwards to persist the updated frontier. + pub fn append( + &mut self, + cmx: [u8; 32], + ciphertext: &TransmittedNoteCiphertext, + ) -> CostResult { + let payload = serialize_ciphertext(ciphertext); + self.append_raw(cmx, &payload) + } + + /// Append a note commitment and raw payload bytes to the commitment tree. + /// + /// Validates that `payload.len() == ciphertext_payload_size::()`. + /// + /// 1. Appends `cmx || payload` to the `BulkAppendTree` (data storage) + /// 2. Appends `cmx` to the Sinsemilla frontier (in-memory) + /// + /// Call [`save`](Self::save) afterwards to persist the updated frontier. + pub fn append_raw( + &mut self, + cmx: [u8; 32], + payload: &[u8], + ) -> CostResult { + let mut cost = OperationCost::default(); + + // Validate payload size + let expected = ciphertext_payload_size::(); + if payload.len() != expected { + return Err(CommitmentTreeError::InvalidPayloadSize { + expected, + actual: payload.len(), + }) + .wrap_with_cost(cost); + } + + // 1. Build cmx||payload and append to BulkAppendTree + let mut item_value = Vec::with_capacity(32 + payload.len()); + item_value.extend_from_slice(&cmx); + item_value.extend_from_slice(payload); + + let bulk_result = match self.bulk_tree.append(&item_value) { + Ok(r) => r, + Err(e) => { + return Err(CommitmentTreeError::InvalidData(format!( + "bulk append: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + cost.hash_node_calls += bulk_result.hash_count; + + // 2. Append cmx to Sinsemilla frontier (tracks sinsemilla_hash_calls) + let sinsemilla_root = match self.frontier.append(cmx) { + grovedb_costs::CostContext { + value: Ok(root), + cost: frontier_cost, + } => { + cost = cost + frontier_cost; + root + } + grovedb_costs::CostContext { + value: Err(e), + cost: frontier_cost, + } => { + cost = cost + frontier_cost; + return Err(e).wrap_with_cost(cost); + } + }; + + Ok(CommitmentAppendResult { + sinsemilla_root, + bulk_state_root: bulk_result.state_root, + global_position: bulk_result.global_position, + hash_count: bulk_result.hash_count, + compacted: bulk_result.compacted, + }) + .wrap_with_cost(cost) + } + + /// Persist the current frontier state to storage. + pub fn save(&self) -> CostResult<(), CommitmentTreeError> { + let mut cost = OperationCost::default(); + let serialized = self.frontier.serialize(); + let result = self + .bulk_tree + .dense_tree + .storage + .put(COMMITMENT_TREE_DATA_KEY, &serialized, None, None) + .unwrap_add_cost(&mut cost); + match result { + Ok(()) => Ok(()).wrap_with_cost(cost), + Err(e) => Err(CommitmentTreeError::InvalidData(format!( + "storage error saving frontier: {}", + e + ))) + .wrap_with_cost(cost), + } + } + + // ── Frontier accessors ──────────────────────────────────────────── + + /// Get the current Sinsemilla root hash as 32 bytes. + pub fn root_hash(&self) -> [u8; 32] { + self.frontier.root_hash() + } + + /// Get the current root as an Orchard `Anchor`. + pub fn anchor(&self) -> crate::Anchor { + self.frontier.anchor() + } + + /// Get the position of the most recently appended leaf, or `None` if empty. + pub fn position(&self) -> Option { + self.frontier.position() + } + + /// Get the number of leaves that have been appended to the frontier. + pub fn tree_size(&self) -> u64 { + self.frontier.tree_size() + } + + // ── BulkAppendTree delegates ────────────────────────────────────── + + /// Get the total count of items appended (from the BulkAppendTree). + pub fn total_count(&self) -> u64 { + self.bulk_tree.total_count + } + + /// Compute the current BulkAppendTree state root without modification. + pub fn compute_current_state_root(&self) -> Result<[u8; 32], CommitmentTreeError> { + self.bulk_tree + .compute_current_state_root() + .map_err(|e| CommitmentTreeError::InvalidData(format!("state root: {}", e))) + } + + /// Get a single value from the dense tree buffer by buffer-local position. + pub fn get_buffer_value(&self, position: u16) -> Result>, CommitmentTreeError> { + self.bulk_tree + .get_buffer_value(position) + .map_err(|e| CommitmentTreeError::InvalidData(format!("buffer value: {}", e))) + } + + /// Get a single completed chunk's raw blob by chunk index. + pub fn get_chunk_value( + &self, + chunk_index: u64, + ) -> Result>, CommitmentTreeError> { + self.bulk_tree + .get_chunk_value(chunk_index) + .map_err(|e| CommitmentTreeError::InvalidData(format!("chunk value: {}", e))) + } + + /// The number of entries per completed chunk (epoch). + pub fn epoch_size(&self) -> u64 { + self.bulk_tree.epoch_size() + } + + /// Number of completed chunks in the MMR. + pub fn chunk_count(&self) -> u64 { + self.bulk_tree.chunk_count() + } +} diff --git a/grovedb-commitment-tree/src/commitment_tree/tests.rs b/grovedb-commitment-tree/src/commitment_tree/tests.rs new file mode 100644 index 000000000..21b152041 --- /dev/null +++ b/grovedb-commitment-tree/src/commitment_tree/tests.rs @@ -0,0 +1,728 @@ +#[cfg(test)] +mod storage_tests { + use std::{collections::BTreeMap, marker::PhantomData}; + + use grovedb_bulk_append_tree::BulkAppendTree; + use grovedb_costs::{ + storage_cost::key_value_cost::KeyValueStorageCost, ChildrenSizesWithIsSumTree, CostContext, + CostResult, CostsExt, OperationCost, + }; + use grovedb_storage::StorageContext; + + use crate::{ + commitment_tree::*, CommitmentFrontier, DashMemo, NoteBytesData, TransmittedNoteCiphertext, + }; + + // ── Mock StorageContext with working data storage ───────────────────── + + /// In-memory key-value store implementing `StorageContext`. + /// + /// Only `get` and `put` are functional — the rest are stubs + /// since `CommitmentTree` only uses data storage operations. + struct MockDataStorageContext { + data: std::cell::RefCell, Vec>>, + } + + impl MockDataStorageContext { + fn new() -> Self { + Self { + data: std::cell::RefCell::new(BTreeMap::new()), + } + } + + /// Create a context pre-seeded with raw bytes at the given key. + fn with_raw_data(key: &[u8], value: Vec) -> Self { + let mut data = BTreeMap::new(); + data.insert(key.to_vec(), value); + Self { + data: std::cell::RefCell::new(data), + } + } + } + + struct StubBatch; + + impl grovedb_storage::Batch for StubBatch { + fn put>( + &mut self, + _key: K, + _value: &[u8], + _children_sizes: ChildrenSizesWithIsSumTree, + _cost_info: Option, + ) -> Result<(), grovedb_costs::error::Error> { + Ok(()) + } + + fn put_aux>( + &mut self, + _key: K, + _value: &[u8], + _cost_info: Option, + ) -> Result<(), grovedb_costs::error::Error> { + Ok(()) + } + + fn put_root>( + &mut self, + _key: K, + _value: &[u8], + _cost_info: Option, + ) -> Result<(), grovedb_costs::error::Error> { + Ok(()) + } + + fn delete>(&mut self, _key: K, _cost_info: Option) {} + + fn delete_aux>(&mut self, _key: K, _cost_info: Option) { + } + + fn delete_root>( + &mut self, + _key: K, + _cost_info: Option, + ) { + } + } + + struct StubRawIterator; + + impl grovedb_storage::RawIterator for StubRawIterator { + fn seek_to_first(&mut self) -> CostContext<()> { + CostContext { + value: (), + cost: Default::default(), + } + } + + fn seek_to_last(&mut self) -> CostContext<()> { + CostContext { + value: (), + cost: Default::default(), + } + } + + fn seek>(&mut self, _key: K) -> CostContext<()> { + CostContext { + value: (), + cost: Default::default(), + } + } + + fn seek_for_prev>(&mut self, _key: K) -> CostContext<()> { + CostContext { + value: (), + cost: Default::default(), + } + } + + fn next(&mut self) -> CostContext<()> { + CostContext { + value: (), + cost: Default::default(), + } + } + + fn prev(&mut self) -> CostContext<()> { + CostContext { + value: (), + cost: Default::default(), + } + } + + fn value(&self) -> CostContext> { + CostContext { + value: None, + cost: Default::default(), + } + } + + fn key(&self) -> CostContext> { + CostContext { + value: None, + cost: Default::default(), + } + } + + fn valid(&self) -> CostContext { + CostContext { + value: false, + cost: Default::default(), + } + } + } + + impl<'db> StorageContext<'db> for MockDataStorageContext { + type Batch = StubBatch; + type RawIterator = StubRawIterator; + + fn put>( + &self, + key: K, + value: &[u8], + _children_sizes: ChildrenSizesWithIsSumTree, + _cost_info: Option, + ) -> CostResult<(), grovedb_storage::Error> { + self.data + .borrow_mut() + .insert(key.as_ref().to_vec(), value.to_vec()); + Ok(()).wrap_with_cost(OperationCost { + seek_count: 1, + ..Default::default() + }) + } + + fn get>( + &self, + key: K, + ) -> CostResult>, grovedb_storage::Error> { + let store = self.data.borrow(); + let val = store.get(key.as_ref()).cloned(); + let loaded = val.as_ref().map_or(0, |v| v.len() as u64); + Ok(val).wrap_with_cost(OperationCost { + seek_count: 1, + storage_loaded_bytes: loaded, + ..Default::default() + }) + } + + fn put_aux>( + &self, + _key: K, + _value: &[u8], + _cost_info: Option, + ) -> CostResult<(), grovedb_storage::Error> { + Ok(()).wrap_with_cost(Default::default()) + } + + fn put_root>( + &self, + _key: K, + _value: &[u8], + _cost_info: Option, + ) -> CostResult<(), grovedb_storage::Error> { + Ok(()).wrap_with_cost(Default::default()) + } + + fn put_meta>( + &self, + _key: K, + _value: &[u8], + _cost_info: Option, + ) -> CostResult<(), grovedb_storage::Error> { + Ok(()).wrap_with_cost(Default::default()) + } + + fn delete>( + &self, + _key: K, + _cost_info: Option, + ) -> CostResult<(), grovedb_storage::Error> { + Ok(()).wrap_with_cost(Default::default()) + } + + fn delete_aux>( + &self, + _key: K, + _cost_info: Option, + ) -> CostResult<(), grovedb_storage::Error> { + Ok(()).wrap_with_cost(Default::default()) + } + + fn delete_root>( + &self, + _key: K, + _cost_info: Option, + ) -> CostResult<(), grovedb_storage::Error> { + Ok(()).wrap_with_cost(Default::default()) + } + + fn delete_meta>( + &self, + _key: K, + _cost_info: Option, + ) -> CostResult<(), grovedb_storage::Error> { + Ok(()).wrap_with_cost(Default::default()) + } + + fn get_aux>( + &self, + _key: K, + ) -> CostResult>, grovedb_storage::Error> { + Ok(None).wrap_with_cost(Default::default()) + } + + fn get_root>( + &self, + _key: K, + ) -> CostResult>, grovedb_storage::Error> { + Ok(None).wrap_with_cost(Default::default()) + } + + fn get_meta>( + &self, + _key: K, + ) -> CostResult>, grovedb_storage::Error> { + Ok(None).wrap_with_cost(Default::default()) + } + + fn new_batch(&self) -> Self::Batch { + StubBatch + } + + fn commit_batch(&self, _batch: Self::Batch) -> CostResult<(), grovedb_storage::Error> { + Ok(()).wrap_with_cost(Default::default()) + } + + fn raw_iter(&self) -> Self::RawIterator { + StubRawIterator + } + } + + // ── Failing mock for error paths ──────────────────────────────────── + + /// Mock StorageContext that returns errors for get and put. + struct FailingDataStorageContext; + + impl<'db> StorageContext<'db> for FailingDataStorageContext { + type Batch = StubBatch; + type RawIterator = StubRawIterator; + + fn get>( + &self, + _key: K, + ) -> CostResult>, grovedb_storage::Error> { + Err(grovedb_storage::Error::StorageError("get failed".into())) + .wrap_with_cost(Default::default()) + } + + fn put>( + &self, + _key: K, + _value: &[u8], + _c: ChildrenSizesWithIsSumTree, + _i: Option, + ) -> CostResult<(), grovedb_storage::Error> { + Err(grovedb_storage::Error::StorageError("put failed".into())) + .wrap_with_cost(Default::default()) + } + + fn get_aux>( + &self, + _key: K, + ) -> CostResult>, grovedb_storage::Error> { + Ok(None).wrap_with_cost(Default::default()) + } + + fn put_aux>( + &self, + _key: K, + _value: &[u8], + _cost_info: Option, + ) -> CostResult<(), grovedb_storage::Error> { + Ok(()).wrap_with_cost(Default::default()) + } + + fn put_root>( + &self, + _k: K, + _v: &[u8], + _c: Option, + ) -> CostResult<(), grovedb_storage::Error> { + Ok(()).wrap_with_cost(Default::default()) + } + + fn put_meta>( + &self, + _k: K, + _v: &[u8], + _c: Option, + ) -> CostResult<(), grovedb_storage::Error> { + Ok(()).wrap_with_cost(Default::default()) + } + + fn delete>( + &self, + _k: K, + _c: Option, + ) -> CostResult<(), grovedb_storage::Error> { + Ok(()).wrap_with_cost(Default::default()) + } + + fn delete_aux>( + &self, + _k: K, + _c: Option, + ) -> CostResult<(), grovedb_storage::Error> { + Ok(()).wrap_with_cost(Default::default()) + } + + fn delete_root>( + &self, + _k: K, + _c: Option, + ) -> CostResult<(), grovedb_storage::Error> { + Ok(()).wrap_with_cost(Default::default()) + } + + fn delete_meta>( + &self, + _k: K, + _c: Option, + ) -> CostResult<(), grovedb_storage::Error> { + Ok(()).wrap_with_cost(Default::default()) + } + + fn get_root>( + &self, + _k: K, + ) -> CostResult>, grovedb_storage::Error> { + Ok(None).wrap_with_cost(Default::default()) + } + + fn get_meta>( + &self, + _k: K, + ) -> CostResult>, grovedb_storage::Error> { + Ok(None).wrap_with_cost(Default::default()) + } + + fn new_batch(&self) -> Self::Batch { + StubBatch + } + + fn commit_batch(&self, _batch: Self::Batch) -> CostResult<(), grovedb_storage::Error> { + Ok(()).wrap_with_cost(Default::default()) + } + + fn raw_iter(&self) -> Self::RawIterator { + StubRawIterator + } + } + + // ── Helpers ───────────────────────────────────────────────────────── + + /// Create a deterministic test leaf from an index. + fn test_leaf(index: u64) -> [u8; 32] { + use incrementalmerkletree::{Hashable, Level}; + use orchard::tree::MerkleHashOrchard; + + let empty = MerkleHashOrchard::empty_leaf(); + let varied = + MerkleHashOrchard::combine(Level::from((index % 31) as u8 + 1), &empty, &empty); + MerkleHashOrchard::combine(Level::from(0), &empty, &varied).to_bytes() + } + + /// Create a deterministic test ciphertext for DashMemo from an index. + /// + /// Layout: `epk_bytes (32) || enc_ciphertext (104) || out_ciphertext (80)` + /// = 216 bytes. + fn test_ciphertext(index: u8) -> TransmittedNoteCiphertext { + let mut epk_bytes = [0u8; 32]; + epk_bytes[0] = index; + epk_bytes[31] = 0xEE; + epk_bytes[1] = index.wrapping_add(1); + + let mut enc_data = [0u8; 104]; + enc_data[0] = index; + enc_data[1] = 0xEC; + let enc_ciphertext = NoteBytesData(enc_data); + + let mut out_ciphertext = [0u8; 80]; + out_ciphertext[0] = index; + out_ciphertext[1] = 0x0C; + + TransmittedNoteCiphertext::from_parts(epk_bytes, enc_ciphertext, out_ciphertext) + } + + /// Default chunk_power for tests (height=1 → capacity=1, epoch_size=2). + const TEST_CHUNK_POWER: u8 = 1; + + // ── Tests ─────────────────────────────────────────────────────────── + + #[test] + fn test_open_empty_store() { + let ctx = MockDataStorageContext::new(); + let result = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx); + let ct = result.value.expect("open should succeed on empty store"); + + assert_eq!( + ct.position(), + None, + "empty frontier should have no position" + ); + assert_eq!(ct.tree_size(), 0, "empty frontier should have size 0"); + assert_eq!(ct.total_count(), 0, "total_count should be 0"); + assert!( + result.cost.seek_count > 0, + "open should report non-zero seek_count" + ); + } + + #[test] + fn test_save_and_load_roundtrip() { + let ctx = MockDataStorageContext::new(); + + // Build a frontier with several leaves, save, then re-open + let result = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx); + let mut ct = result.value.expect("open should succeed"); + for i in 0..20u64 { + ct.append(test_leaf(i), &test_ciphertext(i as u8)) + .value + .expect("append should succeed"); + } + let expected_root = ct.root_hash(); + let expected_position = ct.position(); + let expected_size = ct.tree_size(); + let expected_total_count = ct.total_count(); + + // Save + let save_result = ct.save(); + save_result.value.expect("save should succeed"); + assert!( + save_result.cost.seek_count > 0, + "save should report non-zero seek_count" + ); + + // Re-open from the same storage (extract from bulk tree) + let storage = ct.bulk_tree.dense_tree.storage; + let load_result = + CommitmentTree::<_, DashMemo>::open(expected_total_count, TEST_CHUNK_POWER, storage); + let loaded = load_result.value.expect("open should succeed"); + + assert_eq!(loaded.root_hash(), expected_root, "root hash should match"); + assert_eq!( + loaded.position(), + expected_position, + "position should match" + ); + assert_eq!(loaded.tree_size(), expected_size, "tree size should match"); + assert!( + load_result.cost.storage_loaded_bytes > 0, + "open should report non-zero loaded bytes" + ); + } + + #[test] + fn test_save_overwrite_and_load() { + let ctx = MockDataStorageContext::new(); + let mut ct = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx) + .value + .expect("open should succeed"); + + // Save empty + ct.save().value.expect("save empty should succeed"); + + // Append and save again (overwrites) + ct.append(test_leaf(0), &test_ciphertext(0)) + .value + .expect("append should succeed"); + let expected_root = ct.root_hash(); + let total_count = ct.total_count(); + ct.save().value.expect("save non-empty should succeed"); + + // Re-open should return the latest (non-empty) frontier + let storage = ct.bulk_tree.dense_tree.storage; + let loaded = CommitmentTree::<_, DashMemo>::open(total_count, TEST_CHUNK_POWER, storage) + .value + .expect("open should succeed"); + assert_eq!( + loaded.root_hash(), + expected_root, + "should load the overwritten frontier" + ); + } + + #[test] + fn test_open_corrupted_data_returns_error() { + let ctx = + MockDataStorageContext::with_raw_data(COMMITMENT_TREE_DATA_KEY, vec![0x01, 0x02, 0x03]); + let result = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx); + assert!( + result.value.is_err(), + "should return error for corrupted data" + ); + } + + #[test] + fn test_open_storage_error_surfaces() { + let ctx = FailingDataStorageContext; + let result = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx); + assert!(result.value.is_err(), "should surface storage get error"); + let err_msg = format!("{}", result.value.expect_err("should be storage error")); + assert!( + err_msg.contains("storage error loading frontier"), + "error should contain context: {}", + err_msg + ); + } + + #[test] + fn test_save_storage_error_surfaces() { + // FailingDataStorageContext.get fails, so open() would fail. + // Construct directly to test save() error path. + let bulk_tree = BulkAppendTree::new(TEST_CHUNK_POWER, FailingDataStorageContext) + .expect("bulk tree new should succeed"); + let ct: CommitmentTree<_, DashMemo> = CommitmentTree { + frontier: CommitmentFrontier::new(), + bulk_tree, + _memo: PhantomData, + }; + let result = ct.save(); + assert!(result.value.is_err(), "should surface storage put error"); + let err_msg = format!("{}", result.value.expect_err("should be storage error")); + assert!( + err_msg.contains("storage error saving frontier"), + "error should contain context: {}", + err_msg + ); + } + + #[test] + fn test_save_empty_and_reopen() { + let ctx = MockDataStorageContext::new(); + let ct = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx) + .value + .expect("open should succeed"); + + ct.save().value.expect("save empty should succeed"); + + let storage = ct.bulk_tree.dense_tree.storage; + let loaded = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, storage) + .value + .expect("open should succeed"); + assert_eq!( + loaded.position(), + None, + "loaded empty should have no position" + ); + assert_eq!( + loaded.root_hash(), + CommitmentFrontier::new().root_hash(), + "root hash should match" + ); + } + + #[test] + fn test_roundtrip_with_many_leaves() { + let ctx = MockDataStorageContext::new(); + let mut ct = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx) + .value + .expect("open should succeed"); + + for i in 0..500u64 { + ct.append(test_leaf(i), &test_ciphertext(i as u8)) + .value + .expect("append should succeed"); + } + + let total_count = ct.total_count(); + ct.save().value.expect("save should succeed"); + + let storage = ct.bulk_tree.dense_tree.storage; + let loaded = CommitmentTree::<_, DashMemo>::open(total_count, TEST_CHUNK_POWER, storage) + .value + .expect("open should succeed"); + + // Build an identical frontier to compare root hashes + let mut expected = CommitmentFrontier::new(); + for i in 0..500u64 { + expected + .append(test_leaf(i)) + .value + .expect("append should succeed"); + } + assert_eq!(loaded.root_hash(), expected.root_hash()); + assert_eq!(loaded.tree_size(), 500); + assert_eq!(loaded.position(), Some(499)); + } + + #[test] + fn test_append_returns_result_with_position() { + let ctx = MockDataStorageContext::new(); + let mut ct = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx) + .value + .expect("open should succeed"); + + let r0 = ct + .append(test_leaf(0), &test_ciphertext(0)) + .value + .expect("first append"); + assert_eq!(r0.global_position, 0, "first append should be position 0"); + assert_ne!(r0.sinsemilla_root, [0u8; 32], "root should be non-zero"); + assert_ne!( + r0.bulk_state_root, [0u8; 32], + "state root should be non-zero" + ); + + let r1 = ct + .append(test_leaf(1), &test_ciphertext(1)) + .value + .expect("second append"); + assert_eq!(r1.global_position, 1, "second append should be position 1"); + assert_ne!( + r1.sinsemilla_root, r0.sinsemilla_root, + "roots should differ" + ); + } + + #[test] + fn test_new_creates_empty_tree() { + let ctx = MockDataStorageContext::new(); + let ct = + CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, ctx).expect("new should succeed"); + + assert_eq!(ct.position(), None); + assert_eq!(ct.tree_size(), 0); + assert_eq!(ct.total_count(), 0); + } + + #[test] + fn test_append_raw_rejects_wrong_payload_size() { + let ctx = MockDataStorageContext::new(); + let mut ct = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx) + .value + .expect("open should succeed"); + + // Too small + let result = ct.append_raw(test_leaf(0), &[0u8; 10]); + let err = result.value.expect_err("should reject wrong size"); + let msg = format!("{}", err); + assert!( + msg.contains("invalid payload size"), + "error message should mention payload size: {}", + msg + ); + + // Too large + let result = ct.append_raw(test_leaf(0), &[0u8; 300]); + assert!( + result.value.is_err(), + "should reject payload that is too large" + ); + + // Exact correct size should succeed + let expected_size = ciphertext_payload_size::(); + let result = ct.append_raw(test_leaf(0), &vec![0u8; expected_size]); + assert!(result.value.is_ok(), "correct size should succeed"); + } + + #[test] + fn test_serialize_deserialize_ciphertext_roundtrip() { + let ct = test_ciphertext(42); + let bytes = serialize_ciphertext(&ct); + assert_eq!( + bytes.len(), + ciphertext_payload_size::(), + "serialized size should match expected" + ); + + let deserialized: TransmittedNoteCiphertext = + deserialize_ciphertext(&bytes).expect("deserialization should succeed"); + assert_eq!(deserialized.epk_bytes, ct.epk_bytes); + assert_eq!( + deserialized.enc_ciphertext.as_ref(), + ct.enc_ciphertext.as_ref() + ); + assert_eq!(deserialized.out_ciphertext, ct.out_ciphertext); + } +} diff --git a/grovedb-commitment-tree/src/error.rs b/grovedb-commitment-tree/src/error.rs new file mode 100644 index 000000000..3f1ebbbdd --- /dev/null +++ b/grovedb-commitment-tree/src/error.rs @@ -0,0 +1,15 @@ +use orchard::NOTE_COMMITMENT_TREE_DEPTH; +use thiserror::Error; + +/// Errors that can occur during commitment tree operations. +#[derive(Debug, Error)] +pub enum CommitmentTreeError { + #[error("tree is full (max {max} leaves)", max = 1u64 << NOTE_COMMITMENT_TREE_DEPTH)] + TreeFull, + #[error("invalid frontier data: {0}")] + InvalidData(String), + #[error("invalid Pallas field element")] + InvalidFieldElement, + #[error("invalid payload size: expected {expected}, got {actual}")] + InvalidPayloadSize { expected: usize, actual: usize }, +} diff --git a/grovedb-commitment-tree/src/lib.rs b/grovedb-commitment-tree/src/lib.rs new file mode 100644 index 000000000..70b125106 --- /dev/null +++ b/grovedb-commitment-tree/src/lib.rs @@ -0,0 +1,107 @@ +//! Orchard-style commitment tree integration for GroveDB. +//! +//! This crate provides a lightweight frontier-based Sinsemilla Merkle tree +//! for tracking note commitment anchors. It wraps the `incrementalmerkletree` +//! `Frontier` type with `orchard::tree::MerkleHashOrchard` hashing. +//! +//! # Architecture +//! +//! - Uses `Frontier` for O(1) append and root +//! computation +//! - Stores only the rightmost path (~1KB constant size) rather than the full +//! tree +//! - Items (cmx || encrypted_note) are stored as GroveDB CountTree items +//! - The frontier is serialized to data storage alongside the BulkAppendTree +//! - Historical anchors are managed by Platform in a separate tree (not here) + +#[cfg(feature = "client")] +mod client; +#[cfg(feature = "client")] +pub use client::ClientMemoryCommitmentTree; +mod commitment_frontier; +#[cfg(feature = "storage")] +mod commitment_tree; +mod error; +// Trial decryption functions and traits +#[cfg(feature = "sqlite")] +pub use client::ClientPersistentCommitmentTree; +#[cfg(feature = "sqlite")] +pub use client::{SqliteShardStore, SqliteShardStoreError}; +pub use commitment_frontier::*; +#[cfg(feature = "storage")] +pub use commitment_tree::{ + ciphertext_payload_size, deserialize_ciphertext, serialize_ciphertext, CommitmentAppendResult, + CommitmentTree, COMMITMENT_TREE_DATA_KEY, +}; +pub use error::CommitmentTreeError; +#[cfg(feature = "storage")] +pub use grovedb_bulk_append_tree::{ + deserialize_chunk_blob, serialize_chunk_blob, BulkAppendError, BulkAppendTree, +}; +pub use grovedb_costs::{self}; +pub use incrementalmerkletree::{Hashable, Level, Position, Retention}; +// Builder for constructing shielded transactions +pub use orchard::builder::{Builder, BundleType}; +/// Re-export of `orchard::bundle::BatchValidator` for verifying Orchard +/// bundles. +/// +/// # Sighash Requirement +/// +/// [`BatchValidator::add_bundle`] requires a `sighash: [u8; 32]` parameter — +/// the transaction hash that the Orchard bundle commits to. This hash covers +/// the transaction data excluding the Orchard bundle itself and is used to +/// verify both spend authorization signatures and the binding signature. +/// +/// Platform **must** compute the sighash according to the Dash-adapted +/// equivalent of ZIP-244's transaction digest algorithm and pass it when adding +/// each bundle. Without the correct sighash, signature verification will fail +/// even if the ZK proofs are valid. +/// +/// # Usage +/// +/// ```ignore +/// use grovedb_commitment_tree::{BatchValidator, VerifyingKey}; +/// use rand::rngs::OsRng; +/// +/// let mut validator = BatchValidator::new(); +/// // sighash must be the transaction digest for this bundle +/// validator.add_bundle(&bundle, sighash); +/// // Validate all accumulated bundles (ZK proofs + signatures) +/// let valid = validator.validate(&verifying_key, OsRng); +/// ``` +pub use orchard::bundle::BatchValidator; +// Bundle/Action types +pub use orchard::bundle::{Authorized, Flags}; +// Proof creation/verification (requires orchard "circuit" feature) +pub use orchard::circuit::{ProvingKey, VerifyingKey}; +// Key management +pub use orchard::keys::{ + FullViewingKey, IncomingViewingKey, OutgoingViewingKey, PreparedIncomingViewingKey, Scope, + SpendAuthorizingKey, SpendValidatingKey, SpendingKey, +}; +// Compact note size constant (52 bytes, same for all memo sizes) +pub use orchard::memo::COMPACT_NOTE_SIZE; +// Memo size types for Dash 36-byte memos +pub use orchard::memo::{DashMemo, MemoSize}; +// Note types (orchard::Address aliased to avoid conflict with incrementalmerkletree::Address) +pub use orchard::note::RandomSeed; +// Bundle reconstruction types (needed for deserializing bundles from bytes) +pub use orchard::note::TransmittedNoteCiphertext; +// Orchard tree types +pub use orchard::note::{ExtractedNoteCommitment, Nullifier}; +// Note encryption / trial decryption +pub use orchard::note_encryption::{CompactAction, OrchardDomain}; +// Byte wrapper and trait for constructing note ciphertexts +pub use orchard::zcash_note_encryption::note_bytes::{NoteBytes, NoteBytesData}; +pub use orchard::{ + note::Rho, + primitives::redpallas, + tree::{Anchor, MerkleHashOrchard, MerklePath}, + value::{NoteValue, ValueCommitment}, + zcash_note_encryption::{ + try_compact_note_decryption, try_note_decryption, Domain, EphemeralKeyBytes, ShieldedOutput, + }, + Action, Address as PaymentAddress, Bundle, Note, Proof, NOTE_COMMITMENT_TREE_DEPTH, +}; +#[cfg(feature = "sqlite")] +pub use rusqlite; From f922f28d3515b60227d340d7f2f7633db5ea8455 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 26 Feb 2026 17:23:24 +0700 Subject: [PATCH 2/6] fix: address audit findings and add sinsemilla_hash_calls to OperationCost literals - Replace bare .unwrap() with .expect() in commitment-tree tests - Split sqlite_store.rs into mod.rs, sql_helpers.rs, tree_serialization.rs - Deduplicate SHARD_HEIGHT and test_leaf into shared locations - Add 12 new coverage tests for deserialize, buffer/chunk, state root - Validate cmx field element before BulkAppendTree mutation (F6) - Add recursion depth limit to tree deserialization (V1) - Wrap SQLite checkpoint ops in unchecked_transaction (V9) - Add frontier vs bulk count validation in CommitmentTree::open() - Fix test_schema_idempotent to actually test idempotency - Fix test_bring_your_own_connection to use shared connection - Add sinsemilla_hash_calls: 0 to all OperationCost struct literals - Fix unused variable/import warnings in grovedb tests Co-Authored-By: Claude Opus 4.6 --- grovedb-commitment-tree/Cargo.toml | 2 +- .../client/client_memory_commitment_tree.rs | 4 +- .../client_persistent_commitment_tree.rs | 8 +- grovedb-commitment-tree/src/client/mod.rs | 6 + .../src/client/sqlite_client_tests.rs | 30 +- .../src/client/sqlite_store.rs | 738 ------------------ .../src/client/sqlite_store/mod.rs | 258 ++++++ .../src/client/sqlite_store/sql_helpers.rs | 392 ++++++++++ .../client/sqlite_store/tree_serialization.rs | 181 +++++ .../src/client/sqlite_store_tests.rs | 9 +- grovedb-commitment-tree/src/client/tests.rs | 61 +- .../src/commitment_frontier/tests.rs | 30 +- .../src/commitment_tree/mod.rs | 41 +- .../src/commitment_tree/tests.rs | 240 +++++- grovedb-commitment-tree/src/lib.rs | 2 + grovedb-commitment-tree/src/test_utils.rs | 14 + .../estimated_costs/average_case_costs.rs | 6 + .../batch/estimated_costs/worst_case_costs.rs | 5 + grovedb/src/batch/multi_insert_cost_tests.rs | 3 + grovedb/src/batch/single_insert_cost_tests.rs | 19 + .../single_sum_item_insert_cost_tests.rs | 11 + grovedb/src/operations/delete/mod.rs | 3 + grovedb/src/operations/insert/mod.rs | 20 + grovedb/src/tests/provable_count_tree_test.rs | 2 +- .../src/tests/test_provable_count_fresh.rs | 4 +- merk/src/element/get.rs | 2 + 26 files changed, 1269 insertions(+), 822 deletions(-) delete mode 100644 grovedb-commitment-tree/src/client/sqlite_store.rs create mode 100644 grovedb-commitment-tree/src/client/sqlite_store/mod.rs create mode 100644 grovedb-commitment-tree/src/client/sqlite_store/sql_helpers.rs create mode 100644 grovedb-commitment-tree/src/client/sqlite_store/tree_serialization.rs create mode 100644 grovedb-commitment-tree/src/test_utils.rs diff --git a/grovedb-commitment-tree/Cargo.toml b/grovedb-commitment-tree/Cargo.toml index 2d7c748e2..463e0396e 100644 --- a/grovedb-commitment-tree/Cargo.toml +++ b/grovedb-commitment-tree/Cargo.toml @@ -15,7 +15,7 @@ default = [] server = [] storage = ["grovedb-storage", "grovedb-bulk-append-tree", "server"] client = ["shardtree"] -sqlite = ["shardtree", "rusqlite"] +sqlite = ["client", "rusqlite"] [dependencies] orchard = { git = "https://github.com/dashpay/orchard.git", rev = "41c8f7169f2683c99cf0e0c63e8d25ec12c47a79", features = ["circuit"] } diff --git a/grovedb-commitment-tree/src/client/client_memory_commitment_tree.rs b/grovedb-commitment-tree/src/client/client_memory_commitment_tree.rs index 4d91fe010..267a10902 100644 --- a/grovedb-commitment-tree/src/client/client_memory_commitment_tree.rs +++ b/grovedb-commitment-tree/src/client/client_memory_commitment_tree.rs @@ -5,11 +5,9 @@ use orchard::{ }; use shardtree::{store::memory::MemoryShardStore, ShardTree}; +use super::SHARD_HEIGHT; use crate::commitment_frontier::{merkle_hash_from_bytes, CommitmentTreeError}; -/// Shard height for the ShardTree. Each shard covers 16 levels. -const SHARD_HEIGHT: u8 = 4; - /// Client-side Orchard commitment tree with full Merkle witness support. /// /// Wraps `ShardTree, 32, 4>` with diff --git a/grovedb-commitment-tree/src/client/client_persistent_commitment_tree.rs b/grovedb-commitment-tree/src/client/client_persistent_commitment_tree.rs index 0ebcafd76..abe14d4d8 100644 --- a/grovedb-commitment-tree/src/client/client_persistent_commitment_tree.rs +++ b/grovedb-commitment-tree/src/client/client_persistent_commitment_tree.rs @@ -34,12 +34,12 @@ use orchard::{ use rusqlite::Connection; use shardtree::ShardTree; -use super::sqlite_store::{SqliteShardStore, SqliteShardStoreError}; +use super::{ + sqlite_store::{SqliteShardStore, SqliteShardStoreError}, + SHARD_HEIGHT, +}; use crate::commitment_frontier::{merkle_hash_from_bytes, CommitmentTreeError}; -/// Shard height for the ShardTree. Each shard covers 16 levels. -const SHARD_HEIGHT: u8 = 4; - /// Persistent Orchard commitment tree backed by SQLite. /// /// Same API as diff --git a/grovedb-commitment-tree/src/client/mod.rs b/grovedb-commitment-tree/src/client/mod.rs index 4bcbc00e7..58f57df27 100644 --- a/grovedb-commitment-tree/src/client/mod.rs +++ b/grovedb-commitment-tree/src/client/mod.rs @@ -10,6 +10,12 @@ //! grovedb-commitment-tree = { version = "4", features = ["client"] } //! ``` +/// Shard height for the ShardTree. Each shard covers 2^SHARD_HEIGHT levels. +/// +/// This value is used by all client tree implementations (memory, persistent) +/// and the SQLite store to ensure consistent shard addressing. +pub(crate) const SHARD_HEIGHT: u8 = 4; + mod client_memory_commitment_tree; pub use client_memory_commitment_tree::ClientMemoryCommitmentTree; diff --git a/grovedb-commitment-tree/src/client/sqlite_client_tests.rs b/grovedb-commitment-tree/src/client/sqlite_client_tests.rs index bc31cf703..c74041905 100644 --- a/grovedb-commitment-tree/src/client/sqlite_client_tests.rs +++ b/grovedb-commitment-tree/src/client/sqlite_client_tests.rs @@ -2,18 +2,11 @@ mod tests { use std::sync::{Arc, Mutex}; - use incrementalmerkletree::{Hashable, Level, Position, Retention}; - use orchard::tree::{Anchor, MerkleHashOrchard}; + use incrementalmerkletree::{Position, Retention}; + use orchard::tree::Anchor; use rusqlite::Connection; - use crate::ClientPersistentCommitmentTree; - - fn test_leaf(index: u64) -> [u8; 32] { - let empty = MerkleHashOrchard::empty_leaf(); - let varied = - MerkleHashOrchard::combine(Level::from((index % 31) as u8 + 1), &empty, &empty); - MerkleHashOrchard::combine(Level::from(0), &empty, &varied).to_bytes() - } + use crate::{test_utils::test_leaf, ClientPersistentCommitmentTree}; fn memory_tree() -> ClientPersistentCommitmentTree { let conn = Connection::open_in_memory().expect("open in-memory sqlite"); @@ -128,12 +121,23 @@ mod tests { ) .expect("insert app data"); - let mut tree = ClientPersistentCommitmentTree::open(conn, 100).expect("open tree"); + // Use shared connection so we can verify app data after tree writes + let arc = Arc::new(Mutex::new(conn)); + let mut tree = ClientPersistentCommitmentTree::open_on_shared_connection(arc.clone(), 100) + .expect("open tree"); tree.append(test_leaf(0), Retention::Marked) .expect("append"); - // We can't directly query the connection since it's owned by the tree, - // but the fact that open() succeeded proves coexistence works. + // Verify app table is still readable after commitment tree writes + let guard = arc.lock().expect("lock"); + let value: String = guard + .query_row( + "SELECT value FROM my_app_data WHERE id = 1", + [], + |row| row.get(0), + ) + .expect("query app data"); + assert_eq!(value, "hello", "app data should survive commitment tree writes"); } #[test] diff --git a/grovedb-commitment-tree/src/client/sqlite_store.rs b/grovedb-commitment-tree/src/client/sqlite_store.rs deleted file mode 100644 index 2d8cea56c..000000000 --- a/grovedb-commitment-tree/src/client/sqlite_store.rs +++ /dev/null @@ -1,738 +0,0 @@ -//! SQLite-backed ShardStore for persistent commitment tree storage. -//! -//! Implements the `shardtree::store::ShardStore` trait using a SQLite database, -//! allowing commitment tree state to persist across application restarts. -//! -//! The store creates 4 tables with a `commitment_tree_` prefix so it can -//! coexist safely in any existing SQLite database. -//! -//! # Connection modes -//! -//! - **Owned**: `SqliteShardStore::new(conn)` takes ownership of a -//! `Connection`. -//! - **Shared**: `SqliteShardStore::new_shared(arc)` shares an -//! `Arc>` with other components (e.g., PMT's `Database`). - -use std::{ - collections::BTreeSet, - sync::{Arc, Mutex}, -}; - -use incrementalmerkletree::{Address, Level, Position}; -use orchard::tree::MerkleHashOrchard; -use rusqlite::{params, Connection, OptionalExtension}; -use shardtree::{ - store::{Checkpoint, ShardStore, TreeState}, - LocatedPrunableTree, LocatedTree, Node, PrunableTree, RetentionFlags, Tree, -}; - -use crate::commitment_frontier::merkle_hash_from_bytes; - -/// Shard height — must match the value used in -/// `ClientPersistentCommitmentTree`. -pub(crate) const SHARD_HEIGHT: u8 = 4; - -/// How the store accesses the SQLite connection. -enum ConnectionHolder { - /// The store owns the connection exclusively. - Owned(Connection), - /// The store shares the connection with other components. - Shared(Arc>), -} - -/// SQLite-backed implementation of `ShardStore` for Orchard commitment trees. -/// -/// Stores shard data, cap, and checkpoints in 4 SQLite tables prefixed with -/// `commitment_tree_`. The tables are created automatically on construction. -/// -/// # Connection modes -/// -/// Use [`new`](Self::new) with an owned `Connection`, or -/// [`new_shared`](Self::new_shared) with an `Arc>` to share -/// one connection with the rest of your application. -pub struct SqliteShardStore { - holder: ConnectionHolder, -} - -/// Errors from the SQLite shard store. -#[derive(Debug)] -pub enum SqliteShardStoreError { - /// An error from the underlying SQLite connection. - Sqlite(rusqlite::Error), - /// A serialization or deserialization error. - Serialization(String), -} - -impl std::fmt::Display for SqliteShardStoreError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Sqlite(e) => write!(f, "sqlite error: {e}"), - Self::Serialization(msg) => write!(f, "serialization error: {msg}"), - } - } -} - -impl std::error::Error for SqliteShardStoreError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - Self::Sqlite(e) => Some(e), - Self::Serialization(_) => None, - } - } -} - -impl From for SqliteShardStoreError { - fn from(e: rusqlite::Error) -> Self { - Self::Sqlite(e) - } -} - -impl SqliteShardStore { - /// Create a store that **owns** the given connection. - /// - /// Creates the required tables if they do not already exist. - pub fn new(conn: Connection) -> Result { - create_tables(&conn)?; - Ok(Self { - holder: ConnectionHolder::Owned(conn), - }) - } - - /// Create a store that **shares** a connection via - /// `Arc>`. - /// - /// This lets you use the same SQLite connection that the rest of your - /// application (e.g., a wallet database) already holds. The store locks the - /// mutex for each individual SQL operation. - /// - /// Creates the required tables if they do not already exist. - pub fn new_shared(conn: Arc>) -> Result { - { - let guard = conn.lock().expect("connection mutex poisoned"); - create_tables(&guard)?; - } - Ok(Self { - holder: ConnectionHolder::Shared(conn), - }) - } - - /// Execute a closure with a reference to the underlying connection. - /// - /// For the `Owned` variant this is a direct borrow. For `Shared` it - /// acquires the mutex for the duration of the closure. - pub(crate) fn with_conn(&self, f: impl FnOnce(&Connection) -> T) -> T { - match &self.holder { - ConnectionHolder::Owned(conn) => f(conn), - ConnectionHolder::Shared(arc) => { - let guard = arc.lock().expect("connection mutex poisoned"); - f(&guard) - } - } - } -} - -/// Create the 4 commitment-tree tables if they don't already exist. -fn create_tables(conn: &Connection) -> Result<(), SqliteShardStoreError> { - conn.execute_batch( - "CREATE TABLE IF NOT EXISTS commitment_tree_shards ( - shard_index INTEGER PRIMARY KEY, - shard_data BLOB NOT NULL - ); - CREATE TABLE IF NOT EXISTS commitment_tree_cap ( - id INTEGER PRIMARY KEY CHECK (id = 0), - cap_data BLOB NOT NULL - ); - CREATE TABLE IF NOT EXISTS commitment_tree_checkpoints ( - checkpoint_id INTEGER PRIMARY KEY, - position INTEGER - ); - CREATE TABLE IF NOT EXISTS commitment_tree_checkpoint_marks_removed ( - checkpoint_id INTEGER NOT NULL, - position INTEGER NOT NULL, - PRIMARY KEY (checkpoint_id, position), - FOREIGN KEY (checkpoint_id) REFERENCES commitment_tree_checkpoints(checkpoint_id) - );", - )?; - Ok(()) -} - -// --------------------------------------------------------------------------- -// SQL helpers — all take &Connection so they can be called from with_conn -// --------------------------------------------------------------------------- - -fn sql_get_shard( - conn: &Connection, - shard_root: Address, -) -> Result>, SqliteShardStoreError> { - let index = shard_root.index() as i64; - let row: Option> = conn - .query_row( - "SELECT shard_data FROM commitment_tree_shards WHERE shard_index = ?1", - params![index], - |row| row.get(0), - ) - .optional()?; - - match row { - None => Ok(None), - Some(data) => { - let mut pos = 0; - let tree = deserialize_tree(&data, &mut pos)?; - let located = LocatedTree::from_parts(shard_root, tree).map_err(|addr| { - SqliteShardStoreError::Serialization(format!( - "tree extends beyond shard root at {addr:?}" - )) - })?; - Ok(Some(located)) - } - } -} - -fn sql_last_shard( - conn: &Connection, -) -> Result>, SqliteShardStoreError> { - let row: Option<(i64, Vec)> = conn - .query_row( - "SELECT shard_index, shard_data FROM commitment_tree_shards ORDER BY shard_index DESC \ - LIMIT 1", - [], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .optional()?; - - match row { - None => Ok(None), - Some((index, data)) => { - let addr = Address::from_parts(Level::from(SHARD_HEIGHT), index as u64); - let mut pos = 0; - let tree = deserialize_tree(&data, &mut pos)?; - let located = LocatedTree::from_parts(addr, tree).map_err(|addr| { - SqliteShardStoreError::Serialization(format!( - "tree extends beyond shard root at {addr:?}" - )) - })?; - Ok(Some(located)) - } - } -} - -fn sql_put_shard( - conn: &Connection, - subtree: &LocatedPrunableTree, -) -> Result<(), SqliteShardStoreError> { - let index = subtree.root_addr().index() as i64; - let data = serialize_tree(subtree.root()); - conn.execute( - "INSERT OR REPLACE INTO commitment_tree_shards (shard_index, shard_data) VALUES (?1, ?2)", - params![index, data], - )?; - Ok(()) -} - -fn sql_get_shard_roots(conn: &Connection) -> Result, SqliteShardStoreError> { - let mut stmt = - conn.prepare("SELECT shard_index FROM commitment_tree_shards ORDER BY shard_index")?; - let rows = stmt.query_map([], |row| { - let index: i64 = row.get(0)?; - Ok(Address::from_parts(Level::from(SHARD_HEIGHT), index as u64)) - })?; - let mut result = Vec::new(); - for addr in rows { - result.push(addr?); - } - Ok(result) -} - -fn sql_truncate_shards(conn: &Connection, shard_index: u64) -> Result<(), SqliteShardStoreError> { - conn.execute( - "DELETE FROM commitment_tree_shards WHERE shard_index >= ?1", - params![shard_index as i64], - )?; - Ok(()) -} - -fn sql_get_cap( - conn: &Connection, -) -> Result, SqliteShardStoreError> { - let row: Option> = conn - .query_row( - "SELECT cap_data FROM commitment_tree_cap WHERE id = 0", - [], - |row| row.get(0), - ) - .optional()?; - - match row { - None => Ok(Tree::empty()), - Some(data) => { - let mut pos = 0; - deserialize_tree(&data, &mut pos) - } - } -} - -fn sql_put_cap( - conn: &Connection, - cap: &PrunableTree, -) -> Result<(), SqliteShardStoreError> { - let data = serialize_tree(cap); - conn.execute( - "INSERT OR REPLACE INTO commitment_tree_cap (id, cap_data) VALUES (0, ?1)", - params![data], - )?; - Ok(()) -} - -fn sql_min_checkpoint_id(conn: &Connection) -> Result, SqliteShardStoreError> { - let row: Option = conn.query_row( - "SELECT MIN(checkpoint_id) FROM commitment_tree_checkpoints", - [], - |row| row.get::<_, Option>(0), - )?; - Ok(row) -} - -fn sql_max_checkpoint_id(conn: &Connection) -> Result, SqliteShardStoreError> { - let row: Option = conn.query_row( - "SELECT MAX(checkpoint_id) FROM commitment_tree_checkpoints", - [], - |row| row.get::<_, Option>(0), - )?; - Ok(row) -} - -fn sql_add_checkpoint( - conn: &Connection, - checkpoint_id: u32, - checkpoint: &Checkpoint, -) -> Result<(), SqliteShardStoreError> { - let position: Option = match checkpoint.tree_state() { - TreeState::Empty => None, - TreeState::AtPosition(pos) => Some(u64::from(pos) as i64), - }; - conn.execute( - "INSERT INTO commitment_tree_checkpoints (checkpoint_id, position) VALUES (?1, ?2)", - params![checkpoint_id, position], - )?; - - for mark_pos in checkpoint.marks_removed() { - conn.execute( - "INSERT INTO commitment_tree_checkpoint_marks_removed (checkpoint_id, position) \ - VALUES (?1, ?2)", - params![checkpoint_id, u64::from(*mark_pos) as i64], - )?; - } - Ok(()) -} - -fn sql_checkpoint_count(conn: &Connection) -> Result { - let count: i64 = conn.query_row( - "SELECT COUNT(*) FROM commitment_tree_checkpoints", - [], - |row| row.get(0), - )?; - Ok(count as usize) -} - -fn sql_get_checkpoint_at_depth( - conn: &Connection, - checkpoint_depth: usize, -) -> Result, SqliteShardStoreError> { - let row: Option<(u32, Option)> = conn - .query_row( - "SELECT checkpoint_id, position FROM commitment_tree_checkpoints ORDER BY \ - checkpoint_id DESC LIMIT 1 OFFSET ?1", - params![checkpoint_depth as i64], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .optional()?; - - match row { - None => Ok(None), - Some((id, pos)) => { - let checkpoint = sql_load_checkpoint(conn, id, pos)?; - Ok(Some((id, checkpoint))) - } - } -} - -fn sql_get_checkpoint( - conn: &Connection, - checkpoint_id: u32, -) -> Result, SqliteShardStoreError> { - let row: Option> = conn - .query_row( - "SELECT position FROM commitment_tree_checkpoints WHERE checkpoint_id = ?1", - params![checkpoint_id], - |row| row.get(0), - ) - .optional()?; - - match row { - None => Ok(None), - Some(pos) => { - let checkpoint = sql_load_checkpoint(conn, checkpoint_id, pos)?; - Ok(Some(checkpoint)) - } - } -} - -fn sql_list_checkpoints( - conn: &Connection, - limit: usize, -) -> Result, SqliteShardStoreError> { - let mut stmt = conn.prepare( - "SELECT checkpoint_id, position FROM commitment_tree_checkpoints ORDER BY checkpoint_id \ - DESC LIMIT ?1", - )?; - let rows: Vec<(u32, Option)> = stmt - .query_map(params![limit as i64], |row| Ok((row.get(0)?, row.get(1)?)))? - .collect::, _>>()?; - - let mut result = Vec::with_capacity(rows.len()); - for (id, pos) in rows { - let checkpoint = sql_load_checkpoint(conn, id, pos)?; - result.push((id, checkpoint)); - } - Ok(result) -} - -fn sql_update_checkpoint_with( - conn: &Connection, - checkpoint_id: u32, - update: F, -) -> Result -where - F: Fn(&mut Checkpoint) -> Result<(), SqliteShardStoreError>, -{ - let existing = sql_get_checkpoint(conn, checkpoint_id)?; - match existing { - None => Ok(false), - Some(mut cp) => { - update(&mut cp)?; - conn.execute( - "DELETE FROM commitment_tree_checkpoint_marks_removed WHERE checkpoint_id = ?1", - params![checkpoint_id], - )?; - let position: Option = match cp.tree_state() { - TreeState::Empty => None, - TreeState::AtPosition(pos) => Some(u64::from(pos) as i64), - }; - conn.execute( - "UPDATE commitment_tree_checkpoints SET position = ?1 WHERE checkpoint_id = ?2", - params![position, checkpoint_id], - )?; - for mark_pos in cp.marks_removed() { - conn.execute( - "INSERT INTO commitment_tree_checkpoint_marks_removed (checkpoint_id, \ - position) VALUES (?1, ?2)", - params![checkpoint_id, u64::from(*mark_pos) as i64], - )?; - } - Ok(true) - } - } -} - -fn sql_remove_checkpoint( - conn: &Connection, - checkpoint_id: u32, -) -> Result<(), SqliteShardStoreError> { - conn.execute( - "DELETE FROM commitment_tree_checkpoint_marks_removed WHERE checkpoint_id = ?1", - params![checkpoint_id], - )?; - conn.execute( - "DELETE FROM commitment_tree_checkpoints WHERE checkpoint_id = ?1", - params![checkpoint_id], - )?; - Ok(()) -} - -fn sql_truncate_checkpoints_retaining( - conn: &Connection, - checkpoint_id: u32, -) -> Result<(), SqliteShardStoreError> { - conn.execute( - "DELETE FROM commitment_tree_checkpoint_marks_removed WHERE checkpoint_id > ?1", - params![checkpoint_id], - )?; - conn.execute( - "DELETE FROM commitment_tree_checkpoints WHERE checkpoint_id > ?1", - params![checkpoint_id], - )?; - conn.execute( - "DELETE FROM commitment_tree_checkpoint_marks_removed WHERE checkpoint_id = ?1", - params![checkpoint_id], - )?; - Ok(()) -} - -/// Load a full Checkpoint (including marks_removed). -fn sql_load_checkpoint( - conn: &Connection, - checkpoint_id: u32, - position: Option, -) -> Result { - let tree_state = match position { - None => TreeState::Empty, - Some(p) => TreeState::AtPosition(Position::from(p as u64)), - }; - - let mut stmt = conn.prepare( - "SELECT position FROM commitment_tree_checkpoint_marks_removed WHERE checkpoint_id = ?1", - )?; - let marks: BTreeSet = stmt - .query_map(params![checkpoint_id], |row| { - let p: i64 = row.get(0)?; - Ok(Position::from(p as u64)) - })? - .collect::, _>>()?; - - Ok(Checkpoint::from_parts(tree_state, marks)) -} - -// --------------------------------------------------------------------------- -// ShardStore trait implementation — delegates to sql_* via with_conn -// --------------------------------------------------------------------------- - -impl ShardStore for SqliteShardStore { - type CheckpointId = u32; - type Error = SqliteShardStoreError; - type H = MerkleHashOrchard; - - fn get_shard( - &self, - shard_root: Address, - ) -> Result>, Self::Error> { - self.with_conn(|conn| sql_get_shard(conn, shard_root)) - } - - fn last_shard(&self) -> Result>, Self::Error> { - self.with_conn(sql_last_shard) - } - - fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { - self.with_conn(|conn| sql_put_shard(conn, &subtree)) - } - - fn get_shard_roots(&self) -> Result, Self::Error> { - self.with_conn(sql_get_shard_roots) - } - - fn truncate_shards(&mut self, shard_index: u64) -> Result<(), Self::Error> { - self.with_conn(|conn| sql_truncate_shards(conn, shard_index)) - } - - fn get_cap(&self) -> Result, Self::Error> { - self.with_conn(sql_get_cap) - } - - fn put_cap(&mut self, cap: PrunableTree) -> Result<(), Self::Error> { - self.with_conn(|conn| sql_put_cap(conn, &cap)) - } - - fn min_checkpoint_id(&self) -> Result, Self::Error> { - self.with_conn(sql_min_checkpoint_id) - } - - fn max_checkpoint_id(&self) -> Result, Self::Error> { - self.with_conn(sql_max_checkpoint_id) - } - - fn add_checkpoint( - &mut self, - checkpoint_id: Self::CheckpointId, - checkpoint: Checkpoint, - ) -> Result<(), Self::Error> { - self.with_conn(|conn| sql_add_checkpoint(conn, checkpoint_id, &checkpoint)) - } - - fn checkpoint_count(&self) -> Result { - self.with_conn(sql_checkpoint_count) - } - - fn get_checkpoint_at_depth( - &self, - checkpoint_depth: usize, - ) -> Result, Self::Error> { - self.with_conn(|conn| sql_get_checkpoint_at_depth(conn, checkpoint_depth)) - } - - fn get_checkpoint( - &self, - checkpoint_id: &Self::CheckpointId, - ) -> Result, Self::Error> { - self.with_conn(|conn| sql_get_checkpoint(conn, *checkpoint_id)) - } - - fn with_checkpoints(&mut self, limit: usize, mut callback: F) -> Result<(), Self::Error> - where - F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, - { - let entries = self.with_conn(|conn| sql_list_checkpoints(conn, limit))?; - for (id, checkpoint) in &entries { - callback(id, checkpoint)?; - } - Ok(()) - } - - fn for_each_checkpoint(&self, limit: usize, mut callback: F) -> Result<(), Self::Error> - where - F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, - { - let entries = self.with_conn(|conn| sql_list_checkpoints(conn, limit))?; - for (id, checkpoint) in &entries { - callback(id, checkpoint)?; - } - Ok(()) - } - - fn update_checkpoint_with( - &mut self, - checkpoint_id: &Self::CheckpointId, - update: F, - ) -> Result - where - F: Fn(&mut Checkpoint) -> Result<(), Self::Error>, - { - self.with_conn(|conn| sql_update_checkpoint_with(conn, *checkpoint_id, update)) - } - - fn remove_checkpoint(&mut self, checkpoint_id: &Self::CheckpointId) -> Result<(), Self::Error> { - self.with_conn(|conn| sql_remove_checkpoint(conn, *checkpoint_id)) - } - - fn truncate_checkpoints_retaining( - &mut self, - checkpoint_id: &Self::CheckpointId, - ) -> Result<(), Self::Error> { - self.with_conn(|conn| sql_truncate_checkpoints_retaining(conn, *checkpoint_id)) - } -} - -// --------------------------------------------------------------------------- -// Tree serialization -// --------------------------------------------------------------------------- - -/// Binary format tags for tree nodes. -const TAG_NIL: u8 = 0x00; -const TAG_LEAF: u8 = 0x01; -const TAG_PARENT: u8 = 0x02; - -/// Serialize a `PrunableTree` to bytes. -/// -/// Format: -/// - `Nil`: `[0x00]` -/// - `Leaf`: `[0x01][hash: 32][flags: 1]` -/// - `Parent`: `[0x02][has_ann: 1][ann?: 32][left][right]` -pub(crate) fn serialize_tree(tree: &PrunableTree) -> Vec { - let mut buf = Vec::new(); - serialize_tree_inner(tree, &mut buf); - buf -} - -fn serialize_tree_inner(tree: &PrunableTree, buf: &mut Vec) { - match &**tree { - Node::Nil => { - buf.push(TAG_NIL); - } - Node::Leaf { - value: (hash, flags), - } => { - buf.push(TAG_LEAF); - buf.extend_from_slice(&hash.to_bytes()); - buf.push(flags.bits()); - } - Node::Parent { ann, left, right } => { - buf.push(TAG_PARENT); - match ann { - Some(arc_hash) => { - buf.push(0x01); - buf.extend_from_slice(&arc_hash.to_bytes()); - } - None => { - buf.push(0x00); - } - } - serialize_tree_inner(left, buf); - serialize_tree_inner(right, buf); - } - } -} - -/// Deserialize a `PrunableTree` from bytes. -pub(crate) fn deserialize_tree( - data: &[u8], - pos: &mut usize, -) -> Result, SqliteShardStoreError> { - if *pos >= data.len() { - return Err(SqliteShardStoreError::Serialization( - "unexpected end of data".to_string(), - )); - } - - let tag = data[*pos]; - *pos += 1; - - match tag { - TAG_NIL => Ok(Tree::empty()), - TAG_LEAF => { - if *pos + 33 > data.len() { - return Err(SqliteShardStoreError::Serialization( - "truncated leaf data".to_string(), - )); - } - let hash_bytes: [u8; 32] = data[*pos..*pos + 32] - .try_into() - .map_err(|_| SqliteShardStoreError::Serialization("bad hash".to_string()))?; - *pos += 32; - let flags_byte = data[*pos]; - *pos += 1; - - let hash = merkle_hash_from_bytes(&hash_bytes).ok_or_else(|| { - SqliteShardStoreError::Serialization( - "invalid Pallas field element in leaf".to_string(), - ) - })?; - let flags = RetentionFlags::from_bits_truncate(flags_byte); - Ok(Tree::leaf((hash, flags))) - } - TAG_PARENT => { - if *pos >= data.len() { - return Err(SqliteShardStoreError::Serialization( - "truncated parent annotation flag".to_string(), - )); - } - let has_ann = data[*pos]; - *pos += 1; - - let ann: Option> = if has_ann == 0x01 { - if *pos + 32 > data.len() { - return Err(SqliteShardStoreError::Serialization( - "truncated parent annotation".to_string(), - )); - } - let ann_bytes: [u8; 32] = data[*pos..*pos + 32] - .try_into() - .map_err(|_| SqliteShardStoreError::Serialization("bad ann".to_string()))?; - *pos += 32; - let hash = merkle_hash_from_bytes(&ann_bytes).ok_or_else(|| { - SqliteShardStoreError::Serialization( - "invalid Pallas field element in annotation".to_string(), - ) - })?; - Some(Arc::new(hash)) - } else { - None - }; - - let left = deserialize_tree(data, pos)?; - let right = deserialize_tree(data, pos)?; - Ok(Tree::parent(ann, left, right)) - } - other => Err(SqliteShardStoreError::Serialization(format!( - "unknown tree node tag: 0x{other:02x}" - ))), - } -} diff --git a/grovedb-commitment-tree/src/client/sqlite_store/mod.rs b/grovedb-commitment-tree/src/client/sqlite_store/mod.rs new file mode 100644 index 000000000..abeb4b55d --- /dev/null +++ b/grovedb-commitment-tree/src/client/sqlite_store/mod.rs @@ -0,0 +1,258 @@ +//! SQLite-backed ShardStore for persistent commitment tree storage. +//! +//! Implements the `shardtree::store::ShardStore` trait using a SQLite database, +//! allowing commitment tree state to persist across application restarts. +//! +//! The store creates 4 tables with a `commitment_tree_` prefix so it can +//! coexist safely in any existing SQLite database. +//! +//! # Connection modes +//! +//! - **Owned**: `SqliteShardStore::new(conn)` takes ownership of a +//! `Connection`. +//! - **Shared**: `SqliteShardStore::new_shared(arc)` shares an +//! `Arc>` with other components (e.g., PMT's `Database`). + +mod sql_helpers; +pub(crate) mod tree_serialization; + +use std::sync::{Arc, Mutex}; + +use incrementalmerkletree::Address; +use orchard::tree::MerkleHashOrchard; +use rusqlite::Connection; +use shardtree::{ + store::{Checkpoint, ShardStore}, + LocatedPrunableTree, PrunableTree, +}; + +use sql_helpers::*; + +// Re-export SHARD_HEIGHT from parent so sql_helpers can use it. +pub(crate) use super::SHARD_HEIGHT; + +/// How the store accesses the SQLite connection. +enum ConnectionHolder { + /// The store owns the connection exclusively. + Owned(Connection), + /// The store shares the connection with other components. + Shared(Arc>), +} + +/// SQLite-backed implementation of `ShardStore` for Orchard commitment trees. +/// +/// Stores shard data, cap, and checkpoints in 4 SQLite tables prefixed with +/// `commitment_tree_`. The tables are created automatically on construction. +/// +/// # Connection modes +/// +/// Use [`new`](Self::new) with an owned `Connection`, or +/// [`new_shared`](Self::new_shared) with an `Arc>` to share +/// one connection with the rest of your application. +pub struct SqliteShardStore { + holder: ConnectionHolder, +} + +/// Errors from the SQLite shard store. +#[derive(Debug)] +pub enum SqliteShardStoreError { + /// An error from the underlying SQLite connection. + Sqlite(rusqlite::Error), + /// A serialization or deserialization error. + Serialization(String), +} + +impl std::fmt::Display for SqliteShardStoreError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Sqlite(e) => write!(f, "sqlite error: {e}"), + Self::Serialization(msg) => write!(f, "serialization error: {msg}"), + } + } +} + +impl std::error::Error for SqliteShardStoreError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Sqlite(e) => Some(e), + Self::Serialization(_) => None, + } + } +} + +impl From for SqliteShardStoreError { + fn from(e: rusqlite::Error) -> Self { + Self::Sqlite(e) + } +} + +impl SqliteShardStore { + /// Create a store that **owns** the given connection. + /// + /// Creates the required tables if they do not already exist. + pub fn new(conn: Connection) -> Result { + create_tables(&conn)?; + Ok(Self { + holder: ConnectionHolder::Owned(conn), + }) + } + + /// Create a store that **shares** a connection via + /// `Arc>`. + /// + /// This lets you use the same SQLite connection that the rest of your + /// application (e.g., a wallet database) already holds. The store locks the + /// mutex for each `ShardStore` trait method call, ensuring that + /// multi-statement operations (like checkpoint add with marks) execute + /// atomically within a single lock acquisition. + /// + /// Creates the required tables if they do not already exist. + pub fn new_shared(conn: Arc>) -> Result { + { + let guard = conn.lock().expect("connection mutex poisoned"); + create_tables(&guard)?; + } + Ok(Self { + holder: ConnectionHolder::Shared(conn), + }) + } + + /// Execute a closure with a reference to the underlying connection. + /// + /// For the `Owned` variant this is a direct borrow. For `Shared` it + /// acquires the mutex for the duration of the closure. + /// + /// # Panics + /// + /// Panics if the shared mutex is poisoned (another thread panicked while + /// holding the lock). A poisoned mutex means the connection may be in an + /// inconsistent state, so recovery is not safe. + pub(crate) fn with_conn(&self, f: impl FnOnce(&Connection) -> T) -> T { + match &self.holder { + ConnectionHolder::Owned(conn) => f(conn), + ConnectionHolder::Shared(arc) => { + let guard = arc.lock().expect("connection mutex poisoned"); + f(&guard) + } + } + } +} + +// --------------------------------------------------------------------------- +// ShardStore trait implementation — delegates to sql_* via with_conn +// --------------------------------------------------------------------------- + +impl ShardStore for SqliteShardStore { + type CheckpointId = u32; + type Error = SqliteShardStoreError; + type H = MerkleHashOrchard; + + fn get_shard( + &self, + shard_root: Address, + ) -> Result>, Self::Error> { + self.with_conn(|conn| sql_get_shard(conn, shard_root)) + } + + fn last_shard(&self) -> Result>, Self::Error> { + self.with_conn(sql_last_shard) + } + + fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { + self.with_conn(|conn| sql_put_shard(conn, &subtree)) + } + + fn get_shard_roots(&self) -> Result, Self::Error> { + self.with_conn(sql_get_shard_roots) + } + + fn truncate_shards(&mut self, shard_index: u64) -> Result<(), Self::Error> { + self.with_conn(|conn| sql_truncate_shards(conn, shard_index)) + } + + fn get_cap(&self) -> Result, Self::Error> { + self.with_conn(sql_get_cap) + } + + fn put_cap(&mut self, cap: PrunableTree) -> Result<(), Self::Error> { + self.with_conn(|conn| sql_put_cap(conn, &cap)) + } + + fn min_checkpoint_id(&self) -> Result, Self::Error> { + self.with_conn(sql_min_checkpoint_id) + } + + fn max_checkpoint_id(&self) -> Result, Self::Error> { + self.with_conn(sql_max_checkpoint_id) + } + + fn add_checkpoint( + &mut self, + checkpoint_id: Self::CheckpointId, + checkpoint: Checkpoint, + ) -> Result<(), Self::Error> { + self.with_conn(|conn| sql_add_checkpoint(conn, checkpoint_id, &checkpoint)) + } + + fn checkpoint_count(&self) -> Result { + self.with_conn(sql_checkpoint_count) + } + + fn get_checkpoint_at_depth( + &self, + checkpoint_depth: usize, + ) -> Result, Self::Error> { + self.with_conn(|conn| sql_get_checkpoint_at_depth(conn, checkpoint_depth)) + } + + fn get_checkpoint( + &self, + checkpoint_id: &Self::CheckpointId, + ) -> Result, Self::Error> { + self.with_conn(|conn| sql_get_checkpoint(conn, *checkpoint_id)) + } + + fn with_checkpoints(&mut self, limit: usize, mut callback: F) -> Result<(), Self::Error> + where + F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, + { + let entries = self.with_conn(|conn| sql_list_checkpoints(conn, limit))?; + for (id, checkpoint) in &entries { + callback(id, checkpoint)?; + } + Ok(()) + } + + fn for_each_checkpoint(&self, limit: usize, mut callback: F) -> Result<(), Self::Error> + where + F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, + { + let entries = self.with_conn(|conn| sql_list_checkpoints(conn, limit))?; + for (id, checkpoint) in &entries { + callback(id, checkpoint)?; + } + Ok(()) + } + + fn update_checkpoint_with( + &mut self, + checkpoint_id: &Self::CheckpointId, + update: F, + ) -> Result + where + F: Fn(&mut Checkpoint) -> Result<(), Self::Error>, + { + self.with_conn(|conn| sql_update_checkpoint_with(conn, *checkpoint_id, update)) + } + + fn remove_checkpoint(&mut self, checkpoint_id: &Self::CheckpointId) -> Result<(), Self::Error> { + self.with_conn(|conn| sql_remove_checkpoint(conn, *checkpoint_id)) + } + + fn truncate_checkpoints_retaining( + &mut self, + checkpoint_id: &Self::CheckpointId, + ) -> Result<(), Self::Error> { + self.with_conn(|conn| sql_truncate_checkpoints_retaining(conn, *checkpoint_id)) + } +} diff --git a/grovedb-commitment-tree/src/client/sqlite_store/sql_helpers.rs b/grovedb-commitment-tree/src/client/sqlite_store/sql_helpers.rs new file mode 100644 index 000000000..cb1a429a2 --- /dev/null +++ b/grovedb-commitment-tree/src/client/sqlite_store/sql_helpers.rs @@ -0,0 +1,392 @@ +//! SQL helper functions for the SQLite shard store. +//! +//! All functions take `&Connection` directly so they can be called from +//! `SqliteShardStore::with_conn`. + +use std::collections::BTreeSet; + +use incrementalmerkletree::{Address, Level, Position}; +use orchard::tree::MerkleHashOrchard; +use rusqlite::{params, Connection, OptionalExtension}; +use shardtree::{ + store::{Checkpoint, TreeState}, + LocatedPrunableTree, LocatedTree, PrunableTree, Tree, +}; + +use super::{ + tree_serialization::{deserialize_tree, serialize_tree}, + SqliteShardStoreError, SHARD_HEIGHT, +}; + +pub(crate) fn create_tables(conn: &Connection) -> Result<(), SqliteShardStoreError> { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS commitment_tree_shards ( + shard_index INTEGER PRIMARY KEY, + shard_data BLOB NOT NULL + ); + CREATE TABLE IF NOT EXISTS commitment_tree_cap ( + id INTEGER PRIMARY KEY CHECK (id = 0), + cap_data BLOB NOT NULL + ); + CREATE TABLE IF NOT EXISTS commitment_tree_checkpoints ( + checkpoint_id INTEGER PRIMARY KEY, + position INTEGER + ); + CREATE TABLE IF NOT EXISTS commitment_tree_checkpoint_marks_removed ( + checkpoint_id INTEGER NOT NULL, + position INTEGER NOT NULL, + PRIMARY KEY (checkpoint_id, position), + FOREIGN KEY (checkpoint_id) REFERENCES commitment_tree_checkpoints(checkpoint_id) + );", + )?; + Ok(()) +} + +pub(crate) fn sql_get_shard( + conn: &Connection, + shard_root: Address, +) -> Result>, SqliteShardStoreError> { + let index = shard_root.index() as i64; + let row: Option> = conn + .query_row( + "SELECT shard_data FROM commitment_tree_shards WHERE shard_index = ?1", + params![index], + |row| row.get(0), + ) + .optional()?; + + match row { + None => Ok(None), + Some(data) => { + let mut pos = 0; + let tree = deserialize_tree(&data, &mut pos)?; + let located = LocatedTree::from_parts(shard_root, tree).map_err(|addr| { + SqliteShardStoreError::Serialization(format!( + "tree extends beyond shard root at {addr:?}" + )) + })?; + Ok(Some(located)) + } + } +} + +pub(crate) fn sql_last_shard( + conn: &Connection, +) -> Result>, SqliteShardStoreError> { + let row: Option<(i64, Vec)> = conn + .query_row( + "SELECT shard_index, shard_data FROM commitment_tree_shards ORDER BY shard_index DESC \ + LIMIT 1", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional()?; + + match row { + None => Ok(None), + Some((index, data)) => { + let addr = Address::from_parts(Level::from(SHARD_HEIGHT), index as u64); + let mut pos = 0; + let tree = deserialize_tree(&data, &mut pos)?; + let located = LocatedTree::from_parts(addr, tree).map_err(|addr| { + SqliteShardStoreError::Serialization(format!( + "tree extends beyond shard root at {addr:?}" + )) + })?; + Ok(Some(located)) + } + } +} + +pub(crate) fn sql_put_shard( + conn: &Connection, + subtree: &LocatedPrunableTree, +) -> Result<(), SqliteShardStoreError> { + let index = subtree.root_addr().index() as i64; + let data = serialize_tree(subtree.root()); + conn.execute( + "INSERT OR REPLACE INTO commitment_tree_shards (shard_index, shard_data) VALUES (?1, ?2)", + params![index, data], + )?; + Ok(()) +} + +pub(crate) fn sql_get_shard_roots( + conn: &Connection, +) -> Result, SqliteShardStoreError> { + let mut stmt = + conn.prepare("SELECT shard_index FROM commitment_tree_shards ORDER BY shard_index")?; + let rows = stmt.query_map([], |row| { + let index: i64 = row.get(0)?; + Ok(Address::from_parts(Level::from(SHARD_HEIGHT), index as u64)) + })?; + let mut result = Vec::new(); + for addr in rows { + result.push(addr?); + } + Ok(result) +} + +pub(crate) fn sql_truncate_shards( + conn: &Connection, + shard_index: u64, +) -> Result<(), SqliteShardStoreError> { + conn.execute( + "DELETE FROM commitment_tree_shards WHERE shard_index >= ?1", + params![shard_index as i64], + )?; + Ok(()) +} + +pub(crate) fn sql_get_cap( + conn: &Connection, +) -> Result, SqliteShardStoreError> { + let row: Option> = conn + .query_row( + "SELECT cap_data FROM commitment_tree_cap WHERE id = 0", + [], + |row| row.get(0), + ) + .optional()?; + + match row { + None => Ok(Tree::empty()), + Some(data) => { + let mut pos = 0; + deserialize_tree(&data, &mut pos) + } + } +} + +pub(crate) fn sql_put_cap( + conn: &Connection, + cap: &PrunableTree, +) -> Result<(), SqliteShardStoreError> { + let data = serialize_tree(cap); + conn.execute( + "INSERT OR REPLACE INTO commitment_tree_cap (id, cap_data) VALUES (0, ?1)", + params![data], + )?; + Ok(()) +} + +pub(crate) fn sql_min_checkpoint_id( + conn: &Connection, +) -> Result, SqliteShardStoreError> { + let row: Option = conn.query_row( + "SELECT MIN(checkpoint_id) FROM commitment_tree_checkpoints", + [], + |row| row.get::<_, Option>(0), + )?; + Ok(row) +} + +pub(crate) fn sql_max_checkpoint_id( + conn: &Connection, +) -> Result, SqliteShardStoreError> { + let row: Option = conn.query_row( + "SELECT MAX(checkpoint_id) FROM commitment_tree_checkpoints", + [], + |row| row.get::<_, Option>(0), + )?; + Ok(row) +} + +pub(crate) fn sql_add_checkpoint( + conn: &Connection, + checkpoint_id: u32, + checkpoint: &Checkpoint, +) -> Result<(), SqliteShardStoreError> { + let tx = conn.unchecked_transaction()?; + let position: Option = match checkpoint.tree_state() { + TreeState::Empty => None, + TreeState::AtPosition(pos) => Some(u64::from(pos) as i64), + }; + tx.execute( + "INSERT INTO commitment_tree_checkpoints (checkpoint_id, position) VALUES (?1, ?2)", + params![checkpoint_id, position], + )?; + + for mark_pos in checkpoint.marks_removed() { + tx.execute( + "INSERT INTO commitment_tree_checkpoint_marks_removed (checkpoint_id, position) \ + VALUES (?1, ?2)", + params![checkpoint_id, u64::from(*mark_pos) as i64], + )?; + } + tx.commit()?; + Ok(()) +} + +pub(crate) fn sql_checkpoint_count(conn: &Connection) -> Result { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM commitment_tree_checkpoints", + [], + |row| row.get(0), + )?; + Ok(count as usize) +} + +pub(crate) fn sql_get_checkpoint_at_depth( + conn: &Connection, + checkpoint_depth: usize, +) -> Result, SqliteShardStoreError> { + let row: Option<(u32, Option)> = conn + .query_row( + "SELECT checkpoint_id, position FROM commitment_tree_checkpoints ORDER BY \ + checkpoint_id DESC LIMIT 1 OFFSET ?1", + params![checkpoint_depth as i64], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional()?; + + match row { + None => Ok(None), + Some((id, pos)) => { + let checkpoint = sql_load_checkpoint(conn, id, pos)?; + Ok(Some((id, checkpoint))) + } + } +} + +pub(crate) fn sql_get_checkpoint( + conn: &Connection, + checkpoint_id: u32, +) -> Result, SqliteShardStoreError> { + let row: Option> = conn + .query_row( + "SELECT position FROM commitment_tree_checkpoints WHERE checkpoint_id = ?1", + params![checkpoint_id], + |row| row.get(0), + ) + .optional()?; + + match row { + None => Ok(None), + Some(pos) => { + let checkpoint = sql_load_checkpoint(conn, checkpoint_id, pos)?; + Ok(Some(checkpoint)) + } + } +} + +pub(crate) fn sql_list_checkpoints( + conn: &Connection, + limit: usize, +) -> Result, SqliteShardStoreError> { + let mut stmt = conn.prepare( + "SELECT checkpoint_id, position FROM commitment_tree_checkpoints ORDER BY checkpoint_id \ + DESC LIMIT ?1", + )?; + let rows: Vec<(u32, Option)> = stmt + .query_map(params![limit as i64], |row| Ok((row.get(0)?, row.get(1)?)))? + .collect::, _>>()?; + + let mut result = Vec::with_capacity(rows.len()); + for (id, pos) in rows { + let checkpoint = sql_load_checkpoint(conn, id, pos)?; + result.push((id, checkpoint)); + } + Ok(result) +} + +pub(crate) fn sql_update_checkpoint_with( + conn: &Connection, + checkpoint_id: u32, + update: F, +) -> Result +where + F: Fn(&mut Checkpoint) -> Result<(), SqliteShardStoreError>, +{ + let existing = sql_get_checkpoint(conn, checkpoint_id)?; + match existing { + None => Ok(false), + Some(mut cp) => { + update(&mut cp)?; + let tx = conn.unchecked_transaction()?; + tx.execute( + "DELETE FROM commitment_tree_checkpoint_marks_removed WHERE checkpoint_id = ?1", + params![checkpoint_id], + )?; + let position: Option = match cp.tree_state() { + TreeState::Empty => None, + TreeState::AtPosition(pos) => Some(u64::from(pos) as i64), + }; + tx.execute( + "UPDATE commitment_tree_checkpoints SET position = ?1 WHERE checkpoint_id = ?2", + params![position, checkpoint_id], + )?; + for mark_pos in cp.marks_removed() { + tx.execute( + "INSERT INTO commitment_tree_checkpoint_marks_removed (checkpoint_id, \ + position) VALUES (?1, ?2)", + params![checkpoint_id, u64::from(*mark_pos) as i64], + )?; + } + tx.commit()?; + Ok(true) + } + } +} + +pub(crate) fn sql_remove_checkpoint( + conn: &Connection, + checkpoint_id: u32, +) -> Result<(), SqliteShardStoreError> { + let tx = conn.unchecked_transaction()?; + tx.execute( + "DELETE FROM commitment_tree_checkpoint_marks_removed WHERE checkpoint_id = ?1", + params![checkpoint_id], + )?; + tx.execute( + "DELETE FROM commitment_tree_checkpoints WHERE checkpoint_id = ?1", + params![checkpoint_id], + )?; + tx.commit()?; + Ok(()) +} + +pub(crate) fn sql_truncate_checkpoints_retaining( + conn: &Connection, + checkpoint_id: u32, +) -> Result<(), SqliteShardStoreError> { + let tx = conn.unchecked_transaction()?; + tx.execute( + "DELETE FROM commitment_tree_checkpoint_marks_removed WHERE checkpoint_id > ?1", + params![checkpoint_id], + )?; + tx.execute( + "DELETE FROM commitment_tree_checkpoints WHERE checkpoint_id > ?1", + params![checkpoint_id], + )?; + tx.execute( + "DELETE FROM commitment_tree_checkpoint_marks_removed WHERE checkpoint_id = ?1", + params![checkpoint_id], + )?; + tx.commit()?; + Ok(()) +} + +/// Load a full Checkpoint (including marks_removed). +fn sql_load_checkpoint( + conn: &Connection, + checkpoint_id: u32, + position: Option, +) -> Result { + let tree_state = match position { + None => TreeState::Empty, + Some(p) => TreeState::AtPosition(Position::from(p as u64)), + }; + + let mut stmt = conn.prepare( + "SELECT position FROM commitment_tree_checkpoint_marks_removed WHERE checkpoint_id = ?1", + )?; + let marks: BTreeSet = stmt + .query_map(params![checkpoint_id], |row| { + let p: i64 = row.get(0)?; + Ok(Position::from(p as u64)) + })? + .collect::, _>>()?; + + Ok(Checkpoint::from_parts(tree_state, marks)) +} diff --git a/grovedb-commitment-tree/src/client/sqlite_store/tree_serialization.rs b/grovedb-commitment-tree/src/client/sqlite_store/tree_serialization.rs new file mode 100644 index 000000000..8f333c9d5 --- /dev/null +++ b/grovedb-commitment-tree/src/client/sqlite_store/tree_serialization.rs @@ -0,0 +1,181 @@ +//! Binary serialization of `PrunableTree` nodes. +//! +//! Format: +//! - `Nil`: `[0x00]` +//! - `Leaf`: `[0x01][hash: 32][flags: 1]` +//! - `Parent`: `[0x02][has_ann: 1][ann?: 32][left][right]` + +use std::sync::Arc; + +use orchard::tree::MerkleHashOrchard; +use shardtree::{Node, PrunableTree, RetentionFlags, Tree}; + +use super::SqliteShardStoreError; +use crate::commitment_frontier::merkle_hash_from_bytes; + +/// Binary format tags for tree nodes. +const TAG_NIL: u8 = 0x00; +const TAG_LEAF: u8 = 0x01; +const TAG_PARENT: u8 = 0x02; + +/// Maximum recursion depth for deserialization. +/// +/// A binary tree of depth 32 has at most 32 levels of nesting in its +/// serialization. We allow a small margin above the shard height. +const MAX_DESERIALIZE_DEPTH: usize = 64; + +/// Serialize a `PrunableTree` to bytes. +pub(crate) fn serialize_tree(tree: &PrunableTree) -> Vec { + let mut buf = Vec::new(); + serialize_tree_inner(tree, &mut buf); + buf +} + +fn serialize_tree_inner(tree: &PrunableTree, buf: &mut Vec) { + match &**tree { + Node::Nil => { + buf.push(TAG_NIL); + } + Node::Leaf { + value: (hash, flags), + } => { + buf.push(TAG_LEAF); + buf.extend_from_slice(&hash.to_bytes()); + buf.push(flags.bits()); + } + Node::Parent { ann, left, right } => { + buf.push(TAG_PARENT); + match ann { + Some(arc_hash) => { + buf.push(0x01); + buf.extend_from_slice(&arc_hash.to_bytes()); + } + None => { + buf.push(0x00); + } + } + serialize_tree_inner(left, buf); + serialize_tree_inner(right, buf); + } + } +} + +/// Deserialize a `PrunableTree` from bytes. +pub(crate) fn deserialize_tree( + data: &[u8], + pos: &mut usize, +) -> Result, SqliteShardStoreError> { + deserialize_tree_bounded(data, pos, 0) +} + +/// Depth-bounded deserialization to prevent stack overflow from malicious input. +fn deserialize_tree_bounded( + data: &[u8], + pos: &mut usize, + depth: usize, +) -> Result, SqliteShardStoreError> { + if depth > MAX_DESERIALIZE_DEPTH { + return Err(SqliteShardStoreError::Serialization(format!( + "tree exceeds maximum nesting depth of {}", + MAX_DESERIALIZE_DEPTH + ))); + } + + if *pos >= data.len() { + return Err(SqliteShardStoreError::Serialization( + "unexpected end of data".to_string(), + )); + } + + let tag = data[*pos]; + *pos += 1; + + match tag { + TAG_NIL => Ok(Tree::empty()), + TAG_LEAF => { + if *pos + 33 > data.len() { + return Err(SqliteShardStoreError::Serialization( + "truncated leaf data".to_string(), + )); + } + let hash_bytes: [u8; 32] = data[*pos..*pos + 32] + .try_into() + .map_err(|_| SqliteShardStoreError::Serialization("bad hash".to_string()))?; + *pos += 32; + let flags_byte = data[*pos]; + *pos += 1; + + let hash = merkle_hash_from_bytes(&hash_bytes).ok_or_else(|| { + SqliteShardStoreError::Serialization( + "invalid Pallas field element in leaf".to_string(), + ) + })?; + let flags = RetentionFlags::from_bits_truncate(flags_byte); + Ok(Tree::leaf((hash, flags))) + } + TAG_PARENT => { + if *pos >= data.len() { + return Err(SqliteShardStoreError::Serialization( + "truncated parent annotation flag".to_string(), + )); + } + let has_ann = data[*pos]; + *pos += 1; + + let ann: Option> = if has_ann == 0x01 { + if *pos + 32 > data.len() { + return Err(SqliteShardStoreError::Serialization( + "truncated parent annotation".to_string(), + )); + } + let ann_bytes: [u8; 32] = data[*pos..*pos + 32] + .try_into() + .map_err(|_| SqliteShardStoreError::Serialization("bad ann".to_string()))?; + *pos += 32; + let hash = merkle_hash_from_bytes(&ann_bytes).ok_or_else(|| { + SqliteShardStoreError::Serialization( + "invalid Pallas field element in annotation".to_string(), + ) + })?; + Some(Arc::new(hash)) + } else { + None + }; + + let left = deserialize_tree_bounded(data, pos, depth + 1)?; + let right = deserialize_tree_bounded(data, pos, depth + 1)?; + Ok(Tree::parent(ann, left, right)) + } + other => Err(SqliteShardStoreError::Serialization(format!( + "unknown tree node tag: 0x{other:02x}" + ))), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_deserialize_exceeding_max_depth() { + // Build a deeply nested Parent chain: TAG_PARENT, no annotation, left=Nil, + // right=recurse + let mut data = Vec::new(); + for _ in 0..MAX_DESERIALIZE_DEPTH + 2 { + data.push(TAG_PARENT); + data.push(0x00); // no annotation + data.push(TAG_NIL); // left = nil + // right continues with next Parent + } + data.push(TAG_NIL); // terminal + + let mut pos = 0; + let result = deserialize_tree(&data, &mut pos); + assert!(result.is_err(), "should reject trees exceeding max depth"); + let msg = format!("{}", result.expect_err("should be depth error")); + assert!( + msg.contains("maximum nesting depth"), + "error should mention depth limit: {msg}" + ); + } +} diff --git a/grovedb-commitment-tree/src/client/sqlite_store_tests.rs b/grovedb-commitment-tree/src/client/sqlite_store_tests.rs index b47016ea8..9ffade1d0 100644 --- a/grovedb-commitment-tree/src/client/sqlite_store_tests.rs +++ b/grovedb-commitment-tree/src/client/sqlite_store_tests.rs @@ -14,7 +14,8 @@ mod tests { }; use crate::client::sqlite_store::{ - deserialize_tree, serialize_tree, SqliteShardStore, SHARD_HEIGHT, + tree_serialization::{deserialize_tree, serialize_tree}, + SqliteShardStore, SHARD_HEIGHT, }; fn test_store() -> SqliteShardStore { @@ -48,7 +49,11 @@ mod tests { #[test] fn test_schema_idempotent() { let conn = Connection::open_in_memory().expect("open in-memory sqlite"); - let _store = SqliteShardStore::new(conn).expect("first create"); + let arc = Arc::new(Mutex::new(conn)); + let _store1 = + SqliteShardStore::new_shared(arc.clone()).expect("first create"); + let _store2 = + SqliteShardStore::new_shared(arc.clone()).expect("second create on same DB"); } #[test] diff --git a/grovedb-commitment-tree/src/client/tests.rs b/grovedb-commitment-tree/src/client/tests.rs index db9ba5ef8..f3b04e23e 100644 --- a/grovedb-commitment-tree/src/client/tests.rs +++ b/grovedb-commitment-tree/src/client/tests.rs @@ -1,46 +1,49 @@ #[cfg(test)] mod tests { - use incrementalmerkletree::{Hashable, Level, Position, Retention}; - use orchard::tree::{Anchor, MerkleHashOrchard}; + use incrementalmerkletree::{Position, Retention}; + use orchard::tree::Anchor; - use crate::ClientMemoryCommitmentTree; - - fn test_leaf(index: u64) -> [u8; 32] { - let empty = MerkleHashOrchard::empty_leaf(); - let varied = - MerkleHashOrchard::combine(Level::from((index % 31) as u8 + 1), &empty, &empty); - MerkleHashOrchard::combine(Level::from(0), &empty, &varied).to_bytes() - } + use crate::{test_utils::test_leaf, ClientMemoryCommitmentTree}; #[test] fn test_empty_tree() { let tree = ClientMemoryCommitmentTree::new(10); - assert_eq!(tree.max_leaf_position().unwrap(), None); - assert_eq!(tree.anchor().unwrap(), Anchor::empty_tree()); + assert_eq!(tree.max_leaf_position().expect("max_leaf_position"), None); + assert_eq!(tree.anchor().expect("anchor"), Anchor::empty_tree()); } #[test] fn test_append_and_position() { let mut tree = ClientMemoryCommitmentTree::new(10); - tree.append(test_leaf(0), Retention::Marked).unwrap(); - assert_eq!(tree.max_leaf_position().unwrap(), Some(Position::from(0))); + tree.append(test_leaf(0), Retention::Marked) + .expect("append 0"); + assert_eq!( + tree.max_leaf_position().expect("max_leaf_position"), + Some(Position::from(0)) + ); - tree.append(test_leaf(1), Retention::Ephemeral).unwrap(); - assert_eq!(tree.max_leaf_position().unwrap(), Some(Position::from(1))); + tree.append(test_leaf(1), Retention::Ephemeral) + .expect("append 1"); + assert_eq!( + tree.max_leaf_position().expect("max_leaf_position"), + Some(Position::from(1)) + ); } #[test] fn test_anchor_changes() { let mut tree = ClientMemoryCommitmentTree::new(10); - let empty_anchor = tree.anchor().unwrap(); + let empty_anchor = tree.anchor().expect("anchor"); - tree.append(test_leaf(0), Retention::Marked).unwrap(); - let anchor1 = tree.anchor().unwrap(); + tree.append(test_leaf(0), Retention::Marked) + .expect("append 0"); + let anchor1 = tree.anchor().expect("anchor"); assert_ne!(empty_anchor, anchor1); - tree.append(test_leaf(1), Retention::Marked).unwrap(); - let anchor2 = tree.anchor().unwrap(); + tree.append(test_leaf(1), Retention::Marked) + .expect("append 1"); + let anchor2 = tree.anchor().expect("anchor"); assert_ne!(anchor1, anchor2); } @@ -49,12 +52,14 @@ mod tests { let mut tree = ClientMemoryCommitmentTree::new(10); // Append a marked leaf so we can witness it - tree.append(test_leaf(0), Retention::Marked).unwrap(); - tree.append(test_leaf(1), Retention::Ephemeral).unwrap(); - tree.checkpoint(1).unwrap(); + tree.append(test_leaf(0), Retention::Marked) + .expect("append 0"); + tree.append(test_leaf(1), Retention::Ephemeral) + .expect("append 1"); + tree.checkpoint(1).expect("checkpoint"); // Witness for position 0 at current state - let path = tree.witness(Position::from(0), 0).unwrap(); + let path = tree.witness(Position::from(0), 0).expect("witness"); assert!(path.is_some(), "should produce witness for marked leaf"); } @@ -71,10 +76,12 @@ mod tests { .append(test_leaf(i)) .value .expect("frontier append"); - client.append(test_leaf(i), Retention::Ephemeral).unwrap(); + client + .append(test_leaf(i), Retention::Ephemeral) + .expect("client append"); } - assert_eq!(frontier.anchor(), client.anchor().unwrap()); + assert_eq!(frontier.anchor(), client.anchor().expect("client anchor")); } /// Demonstrates that `checkpoint()` with a duplicate ID silently returns diff --git a/grovedb-commitment-tree/src/commitment_frontier/tests.rs b/grovedb-commitment-tree/src/commitment_frontier/tests.rs index 3febace53..d2b97acca 100644 --- a/grovedb-commitment-tree/src/commitment_frontier/tests.rs +++ b/grovedb-commitment-tree/src/commitment_frontier/tests.rs @@ -6,18 +6,11 @@ mod tests { NOTE_COMMITMENT_TREE_DEPTH, }; - use crate::commitment_frontier::{ - empty_sinsemilla_root, CommitmentFrontier, EMPTY_SINSEMILLA_ROOT, + use crate::{ + commitment_frontier::{empty_sinsemilla_root, CommitmentFrontier, EMPTY_SINSEMILLA_ROOT}, + test_utils::test_leaf, }; - /// Create a deterministic test leaf from an index. - fn test_leaf(index: u64) -> [u8; 32] { - let empty = MerkleHashOrchard::empty_leaf(); - let varied = - MerkleHashOrchard::combine(Level::from((index % 31) as u8 + 1), &empty, &empty); - MerkleHashOrchard::combine(Level::from(0), &empty, &varied).to_bytes() - } - #[test] fn test_empty_frontier() { let f = CommitmentFrontier::new(); @@ -88,7 +81,7 @@ mod tests { fn test_serialize_empty() { let f = CommitmentFrontier::new(); let data = f.serialize(); - let f2 = CommitmentFrontier::deserialize(&data).unwrap(); + let f2 = CommitmentFrontier::deserialize(&data).expect("deserialize empty frontier"); assert_eq!(f.root_hash(), f2.root_hash()); assert_eq!(f.position(), f2.position()); @@ -102,7 +95,7 @@ mod tests { } let data = f.serialize(); - let f2 = CommitmentFrontier::deserialize(&data).unwrap(); + let f2 = CommitmentFrontier::deserialize(&data).expect("deserialize frontier"); assert_eq!(f.root_hash(), f2.root_hash()); assert_eq!(f.position(), f2.position()); @@ -126,7 +119,8 @@ mod tests { data.len() ); - let f2 = CommitmentFrontier::deserialize(&data).unwrap(); + let f2 = + CommitmentFrontier::deserialize(&data).expect("deserialize frontier with many leaves"); assert_eq!(f.root_hash(), f2.root_hash()); assert_eq!(f.tree_size(), f2.tree_size()); } @@ -201,7 +195,7 @@ mod tests { let truncated = &data[..43]; let err = CommitmentFrontier::deserialize(truncated); assert!(err.is_err(), "should fail on truncated ommers"); - let msg = format!("{}", err.unwrap_err()); + let msg = format!("{}", err.expect_err("should be an error")); assert!( msg.contains("truncated ommers"), "expected 'truncated ommers' error, got: {msg}" @@ -219,7 +213,7 @@ mod tests { let err = CommitmentFrontier::deserialize(&data); assert!(err.is_err(), "should fail on invalid leaf field element"); - let msg = format!("{}", err.unwrap_err()); + let msg = format!("{}", err.expect_err("should be an error")); assert!( msg.contains("invalid Pallas field element"), "expected InvalidFieldElement error, got: {msg}" @@ -242,7 +236,7 @@ mod tests { } let err = CommitmentFrontier::deserialize(&data); assert!(err.is_err(), "should fail on invalid ommer field element"); - let msg = format!("{}", err.unwrap_err()); + let msg = format!("{}", err.expect_err("should be an error")); assert!( msg.contains("invalid Pallas field element"), "expected InvalidFieldElement error, got: {msg}" @@ -266,7 +260,7 @@ mod tests { let err = CommitmentFrontier::deserialize(&data); assert!(err.is_err(), "should fail on inconsistent from_parts"); - let msg = format!("{}", err.unwrap_err()); + let msg = format!("{}", err.expect_err("should be an error")); assert!( msg.contains("frontier reconstruction"), "expected 'frontier reconstruction' error, got: {msg}" @@ -309,7 +303,7 @@ mod tests { // Test with a frontier flag value that is neither 0x00 nor 0x01 let err = CommitmentFrontier::deserialize(&[0x42]); assert!(err.is_err()); - let msg = format!("{}", err.unwrap_err()); + let msg = format!("{}", err.expect_err("should be an error")); assert!( msg.contains("invalid frontier flag: 0x42"), "expected 'invalid frontier flag' error, got: {msg}" diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs index a6379c2e4..f8ac5f7f9 100644 --- a/grovedb-commitment-tree/src/commitment_tree/mod.rs +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -99,9 +99,29 @@ pub fn deserialize_ciphertext(data: &[u8]) -> Option { frontier: CommitmentFrontier, - pub bulk_tree: BulkAppendTree, + pub(crate) bulk_tree: BulkAppendTree, _memo: PhantomData, } @@ -175,6 +195,18 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { } }; + // Validate that the frontier and bulk tree agree on the number of + // appended items. A mismatch indicates a partial commit or data + // corruption. + let frontier_size = frontier.tree_size(); + if frontier_size != total_count { + return Err(CommitmentTreeError::InvalidData(format!( + "frontier tree_size ({}) != bulk tree total_count ({})", + frontier_size, total_count + ))) + .wrap_with_cost(cost); + } + Ok(Self { frontier, bulk_tree, @@ -213,6 +245,13 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { ) -> CostResult { let mut cost = OperationCost::default(); + // Validate cmx is a valid Pallas field element before any mutation. + // This prevents inconsistent state if BulkAppendTree is mutated but + // the frontier rejects the cmx. + if crate::commitment_frontier::merkle_hash_from_bytes(&cmx).is_none() { + return Err(CommitmentTreeError::InvalidFieldElement).wrap_with_cost(cost); + } + // Validate payload size let expected = ciphertext_payload_size::(); if payload.len() != expected { diff --git a/grovedb-commitment-tree/src/commitment_tree/tests.rs b/grovedb-commitment-tree/src/commitment_tree/tests.rs index 21b152041..f4d1583f7 100644 --- a/grovedb-commitment-tree/src/commitment_tree/tests.rs +++ b/grovedb-commitment-tree/src/commitment_tree/tests.rs @@ -10,7 +10,8 @@ mod storage_tests { use grovedb_storage::StorageContext; use crate::{ - commitment_tree::*, CommitmentFrontier, DashMemo, NoteBytesData, TransmittedNoteCiphertext, + commitment_tree::*, test_utils::test_leaf, CommitmentFrontier, DashMemo, NoteBytesData, + TransmittedNoteCiphertext, }; // ── Mock StorageContext with working data storage ───────────────────── @@ -401,17 +402,6 @@ mod storage_tests { // ── Helpers ───────────────────────────────────────────────────────── - /// Create a deterministic test leaf from an index. - fn test_leaf(index: u64) -> [u8; 32] { - use incrementalmerkletree::{Hashable, Level}; - use orchard::tree::MerkleHashOrchard; - - let empty = MerkleHashOrchard::empty_leaf(); - let varied = - MerkleHashOrchard::combine(Level::from((index % 31) as u8 + 1), &empty, &empty); - MerkleHashOrchard::combine(Level::from(0), &empty, &varied).to_bytes() - } - /// Create a deterministic test ciphertext for DashMemo from an index. /// /// Layout: `epk_bytes (32) || enc_ciphertext (104) || out_ciphertext (80)` @@ -725,4 +715,230 @@ mod storage_tests { ); assert_eq!(deserialized.out_ciphertext, ct.out_ciphertext); } + + // ── Coverage gap tests ───────────────────────────────────────────── + + #[test] + fn test_deserialize_ciphertext_too_short() { + // Less than 32 + 80 = 112 bytes minimum + let result: Option> = + deserialize_ciphertext(&[0u8; 50]); + assert!(result.is_none(), "should return None for too-short data"); + } + + #[test] + fn test_deserialize_ciphertext_empty() { + let result: Option> = deserialize_ciphertext(&[]); + assert!(result.is_none(), "should return None for empty data"); + } + + #[test] + fn test_deserialize_ciphertext_wrong_enc_size() { + // 32 (epk) + wrong enc size + 80 (out) = 113 bytes total + // enc_size = 113 - 32 - 80 = 1 byte, but DashMemo expects 104 + let result: Option> = + deserialize_ciphertext(&[0u8; 113]); + assert!( + result.is_none(), + "should return None for wrong enc_ciphertext size" + ); + } + + #[test] + fn test_get_buffer_value_empty_tree() { + let ctx = MockDataStorageContext::new(); + let ct = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx) + .value + .expect("open should succeed"); + + let result = ct + .get_buffer_value(0) + .expect("get_buffer_value should not error"); + assert!(result.is_none(), "empty tree should have no buffer values"); + } + + #[test] + fn test_get_buffer_value_after_appends() { + let ctx = MockDataStorageContext::new(); + let mut ct = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx) + .value + .expect("open should succeed"); + + // Append one item (goes into buffer since epoch_size = 2 for chunk_power=1) + ct.append(test_leaf(0), &test_ciphertext(0)) + .value + .expect("append should succeed"); + + let val = ct + .get_buffer_value(0) + .expect("get_buffer_value should not error"); + assert!(val.is_some(), "buffer should contain the first entry"); + + // Position beyond buffer should be None + let val = ct + .get_buffer_value(100) + .expect("get_buffer_value should not error"); + assert!(val.is_none(), "out-of-range position should return None"); + } + + #[test] + fn test_get_chunk_value_empty_tree() { + let ctx = MockDataStorageContext::new(); + let ct = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx) + .value + .expect("open should succeed"); + + let result = ct + .get_chunk_value(0) + .expect("get_chunk_value should not error"); + assert!(result.is_none(), "empty tree should have no chunks"); + } + + #[test] + fn test_get_chunk_value_after_compaction() { + let ctx = MockDataStorageContext::new(); + let mut ct = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx) + .value + .expect("open should succeed"); + + // chunk_power=1 → epoch_size=2. Append 2 items to trigger compaction. + ct.append(test_leaf(0), &test_ciphertext(0)) + .value + .expect("append 0"); + let r = ct + .append(test_leaf(1), &test_ciphertext(1)) + .value + .expect("append 1"); + assert!(r.compacted, "second append should trigger compaction"); + + let chunk = ct + .get_chunk_value(0) + .expect("get_chunk_value should not error"); + assert!(chunk.is_some(), "chunk 0 should exist after compaction"); + + let no_chunk = ct + .get_chunk_value(99) + .expect("get_chunk_value should not error"); + assert!(no_chunk.is_none(), "non-existent chunk should return None"); + } + + #[test] + fn test_compute_current_state_root_empty() { + let ctx = MockDataStorageContext::new(); + let ct = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx) + .value + .expect("open should succeed"); + + let root = ct + .compute_current_state_root() + .expect("state root should succeed"); + // Empty tree still has a deterministic root + assert_ne!(root, [0u8; 32], "empty state root should be non-zero"); + } + + #[test] + fn test_compute_current_state_root_matches_append_result() { + let ctx = MockDataStorageContext::new(); + let mut ct = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx) + .value + .expect("open should succeed"); + + let r = ct + .append(test_leaf(0), &test_ciphertext(0)) + .value + .expect("append 0"); + + let computed = ct + .compute_current_state_root() + .expect("state root should succeed"); + assert_eq!( + computed, r.bulk_state_root, + "computed state root should match append result" + ); + } + + #[test] + fn test_epoch_size_and_chunk_count() { + let ctx = MockDataStorageContext::new(); + let mut ct = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx) + .value + .expect("open should succeed"); + + assert_eq!(ct.epoch_size(), 2, "chunk_power=1 → epoch_size=2"); + assert_eq!(ct.chunk_count(), 0, "no chunks initially"); + + // Fill one epoch + ct.append(test_leaf(0), &test_ciphertext(0)) + .value + .expect("append 0"); + ct.append(test_leaf(1), &test_ciphertext(1)) + .value + .expect("append 1"); + + assert_eq!(ct.chunk_count(), 1, "one chunk after filling one epoch"); + + // Fill another epoch + ct.append(test_leaf(2), &test_ciphertext(2)) + .value + .expect("append 2"); + ct.append(test_leaf(3), &test_ciphertext(3)) + .value + .expect("append 3"); + + assert_eq!(ct.chunk_count(), 2, "two chunks after filling two epochs"); + } + + #[test] + fn test_anchor_on_commitment_tree() { + let ctx = MockDataStorageContext::new(); + let mut ct = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx) + .value + .expect("open should succeed"); + + let empty_anchor = ct.anchor(); + assert_eq!( + empty_anchor, + crate::Anchor::empty_tree(), + "empty tree should have empty anchor" + ); + + ct.append(test_leaf(0), &test_ciphertext(0)) + .value + .expect("append 0"); + + let anchor = ct.anchor(); + assert_ne!( + anchor, + crate::Anchor::empty_tree(), + "non-empty tree should have non-empty anchor" + ); + } + + #[test] + fn test_append_raw_rejects_invalid_cmx() { + let ctx = MockDataStorageContext::new(); + let mut ct = CommitmentTree::<_, DashMemo>::open(0, TEST_CHUNK_POWER, ctx) + .value + .expect("open should succeed"); + + // All 0xFF is not a valid Pallas field element + let payload = vec![0u8; ciphertext_payload_size::()]; + let result = ct.append_raw([0xFF; 32], &payload); + assert!( + result.value.is_err(), + "should reject invalid cmx field element" + ); + let msg = format!("{}", result.value.expect_err("should be an error")); + assert!( + msg.contains("invalid Pallas field element"), + "error should mention field element: {msg}" + ); + + // Verify tree was NOT mutated + assert_eq!( + ct.total_count(), + 0, + "tree should not have been mutated by invalid cmx" + ); + } } diff --git a/grovedb-commitment-tree/src/lib.rs b/grovedb-commitment-tree/src/lib.rs index 70b125106..09cb3f083 100644 --- a/grovedb-commitment-tree/src/lib.rs +++ b/grovedb-commitment-tree/src/lib.rs @@ -22,6 +22,8 @@ mod commitment_frontier; #[cfg(feature = "storage")] mod commitment_tree; mod error; +#[cfg(test)] +pub(crate) mod test_utils; // Trial decryption functions and traits #[cfg(feature = "sqlite")] pub use client::ClientPersistentCommitmentTree; diff --git a/grovedb-commitment-tree/src/test_utils.rs b/grovedb-commitment-tree/src/test_utils.rs new file mode 100644 index 000000000..f5c9ce1ec --- /dev/null +++ b/grovedb-commitment-tree/src/test_utils.rs @@ -0,0 +1,14 @@ +//! Shared test utilities for the commitment-tree crate. + +use incrementalmerkletree::{Hashable, Level}; +use orchard::tree::MerkleHashOrchard; + +/// Create a deterministic test leaf from an index. +/// +/// Produces a valid Pallas field element (32 bytes) that is unique per index. +/// Uses Sinsemilla `combine` at different levels to produce varied hashes. +pub fn test_leaf(index: u64) -> [u8; 32] { + let empty = MerkleHashOrchard::empty_leaf(); + let varied = MerkleHashOrchard::combine(Level::from((index % 31) as u8 + 1), &empty, &empty); + MerkleHashOrchard::combine(Level::from(0), &empty, &varied).to_bytes() +} diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 13fdbd29c..ff44b0c2d 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -383,6 +383,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 6, + sinsemilla_hash_calls: 0, } ); } @@ -450,6 +451,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 6, + sinsemilla_hash_calls: 0, } ); } @@ -512,6 +514,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 4, + sinsemilla_hash_calls: 0, } ); } @@ -598,6 +601,7 @@ mod tests { }, storage_loaded_bytes: 109, hash_node_calls: 8, + sinsemilla_hash_calls: 0, } ); } @@ -691,6 +695,7 @@ mod tests { }, storage_loaded_bytes: 173, hash_node_calls: 12, + sinsemilla_hash_calls: 0, } ); } @@ -759,6 +764,7 @@ mod tests { }, storage_loaded_bytes: 7669, hash_node_calls: 79, + sinsemilla_hash_calls: 0, } ); } diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 82dd7bbe2..4449f1e02 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -332,6 +332,7 @@ mod tests { }, storage_loaded_bytes: 65791, hash_node_calls: 8, // todo: verify why + sinsemilla_hash_calls: 0, } ); } @@ -387,6 +388,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 6, + sinsemilla_hash_calls: 0, } ); } @@ -442,6 +444,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 4, + sinsemilla_hash_calls: 0, } ); } @@ -508,6 +511,7 @@ mod tests { }, storage_loaded_bytes: 2236894, hash_node_calls: 74, + sinsemilla_hash_calls: 0, } ); } @@ -572,6 +576,7 @@ mod tests { }, storage_loaded_bytes: 65964, hash_node_calls: 266, + sinsemilla_hash_calls: 0, } ); } diff --git a/grovedb/src/batch/multi_insert_cost_tests.rs b/grovedb/src/batch/multi_insert_cost_tests.rs index 370aac91c..d76562339 100644 --- a/grovedb/src/batch/multi_insert_cost_tests.rs +++ b/grovedb/src/batch/multi_insert_cost_tests.rs @@ -286,6 +286,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 12, + sinsemilla_hash_calls: 0, } ); } @@ -352,6 +353,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 12, + sinsemilla_hash_calls: 0, } ); } @@ -450,6 +452,7 @@ mod tests { }, storage_loaded_bytes: 152, // todo: verify this hash_node_calls: 22, // todo: verify this + sinsemilla_hash_calls: 0, } ); diff --git a/grovedb/src/batch/single_insert_cost_tests.rs b/grovedb/src/batch/single_insert_cost_tests.rs index b3750ecae..3aa8a9798 100644 --- a/grovedb/src/batch/single_insert_cost_tests.rs +++ b/grovedb/src/batch/single_insert_cost_tests.rs @@ -113,6 +113,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 6, + sinsemilla_hash_calls: 0, } ); } @@ -177,6 +178,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 4, + sinsemilla_hash_calls: 0, } ); } @@ -273,6 +275,7 @@ mod tests { }, storage_loaded_bytes: 74, // todo: verify and explain hash_node_calls: 8, + sinsemilla_hash_calls: 0, } ); } @@ -355,6 +358,7 @@ mod tests { }, storage_loaded_bytes: 71, // todo: verify and explain hash_node_calls: 8, + sinsemilla_hash_calls: 0, } ); } @@ -444,6 +448,7 @@ mod tests { }, storage_loaded_bytes: 146, // todo: verify and explain hash_node_calls: 12, + sinsemilla_hash_calls: 0, } ); } @@ -507,6 +512,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 4, + sinsemilla_hash_calls: 0, } ); } @@ -570,6 +576,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 4, + sinsemilla_hash_calls: 0, } ); } @@ -635,6 +642,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 4, + sinsemilla_hash_calls: 0, } ); } @@ -700,6 +708,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 4, + sinsemilla_hash_calls: 0, } ); } @@ -766,6 +775,7 @@ mod tests { }, storage_loaded_bytes: 235, // todo: verify this hash_node_calls: 10, // todo: verify this + sinsemilla_hash_calls: 0, } ); } @@ -860,6 +870,7 @@ mod tests { }, storage_loaded_bytes: 236, // todo: verify this hash_node_calls: 10, // todo: verify this + sinsemilla_hash_calls: 0, } ); } @@ -975,6 +986,7 @@ mod tests { }, storage_loaded_bytes: 357, // todo: verify this hash_node_calls: 16, // todo: verify this + sinsemilla_hash_calls: 0, } ); @@ -1093,6 +1105,7 @@ mod tests { }, storage_loaded_bytes: 236, // todo: verify this hash_node_calls: 16, // todo: verify this + sinsemilla_hash_calls: 0, } ); @@ -1172,6 +1185,7 @@ mod tests { }, storage_loaded_bytes: 235, // todo: verify this hash_node_calls: 10, // todo: verify this + sinsemilla_hash_calls: 0, } ); } @@ -1267,6 +1281,7 @@ mod tests { }, storage_loaded_bytes: 236, // todo: verify this hash_node_calls: 10, // todo: verify this + sinsemilla_hash_calls: 0, } ); } @@ -1383,6 +1398,7 @@ mod tests { }, storage_loaded_bytes: 357, // todo: verify this hash_node_calls: 16, // todo: verify this + sinsemilla_hash_calls: 0, } ); @@ -1483,6 +1499,7 @@ mod tests { }, storage_loaded_bytes: 230, // todo: verify this hash_node_calls: 12, // todo: verify this + sinsemilla_hash_calls: 0, } ); } @@ -1575,6 +1592,7 @@ mod tests { }, storage_loaded_bytes: 380, // todo: verify this hash_node_calls: 12, // todo: verify this + sinsemilla_hash_calls: 0, } ); @@ -1674,6 +1692,7 @@ mod tests { }, storage_loaded_bytes: 133, // todo: verify this hash_node_calls: 12, // todo: verify this + sinsemilla_hash_calls: 0, } ); diff --git a/grovedb/src/batch/single_sum_item_insert_cost_tests.rs b/grovedb/src/batch/single_sum_item_insert_cost_tests.rs index ae16eaa2b..2e7bb4be8 100644 --- a/grovedb/src/batch/single_sum_item_insert_cost_tests.rs +++ b/grovedb/src/batch/single_sum_item_insert_cost_tests.rs @@ -113,6 +113,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 6, + sinsemilla_hash_calls: 0, } ); } @@ -199,6 +200,7 @@ mod tests { }, storage_loaded_bytes: 71, // todo: verify and explain hash_node_calls: 8, + sinsemilla_hash_calls: 0, } ); } @@ -285,6 +287,7 @@ mod tests { }, storage_loaded_bytes: 72, // todo: verify and explain hash_node_calls: 8, + sinsemilla_hash_calls: 0, } ); } @@ -375,6 +378,7 @@ mod tests { }, storage_loaded_bytes: 146, // todo: verify and explain hash_node_calls: 12, + sinsemilla_hash_calls: 0, } ); } @@ -466,6 +470,7 @@ mod tests { }, storage_loaded_bytes: 156, // todo: verify and explain hash_node_calls: 12, + sinsemilla_hash_calls: 0, } ); } @@ -541,6 +546,7 @@ mod tests { }, storage_loaded_bytes: 170, hash_node_calls: 10, + sinsemilla_hash_calls: 0, } ); } @@ -616,6 +622,7 @@ mod tests { }, storage_loaded_bytes: 170, hash_node_calls: 10, + sinsemilla_hash_calls: 0, } ); } @@ -682,6 +689,7 @@ mod tests { }, storage_loaded_bytes: 239, // todo: verify this hash_node_calls: 10, // todo: verify this + sinsemilla_hash_calls: 0, } ); } @@ -748,6 +756,7 @@ mod tests { }, storage_loaded_bytes: 241, // todo: verify this hash_node_calls: 10, // todo: verify this + sinsemilla_hash_calls: 0, } ); } @@ -814,6 +823,7 @@ mod tests { }, storage_loaded_bytes: 248, // todo: verify this hash_node_calls: 10, // todo: verify this + sinsemilla_hash_calls: 0, } ); } @@ -880,6 +890,7 @@ mod tests { }, storage_loaded_bytes: 251, // todo: verify this hash_node_calls: 10, // todo: verify this + sinsemilla_hash_calls: 0, } ); } diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index c2b9342d8..3ef2826ef 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -1454,6 +1454,7 @@ mod tests { }, storage_loaded_bytes: 154, // todo: verify this hash_node_calls: 0, + sinsemilla_hash_calls: 0, } ); } @@ -1540,6 +1541,7 @@ mod tests { }, storage_loaded_bytes: 418, // todo: verify this hash_node_calls: 5, + sinsemilla_hash_calls: 0, } ); } @@ -1627,6 +1629,7 @@ mod tests { }, storage_loaded_bytes: 418, // todo: verify this hash_node_calls: 5, + sinsemilla_hash_calls: 0, } ); } diff --git a/grovedb/src/operations/insert/mod.rs b/grovedb/src/operations/insert/mod.rs index 5761f50b3..c29c20c4b 100644 --- a/grovedb/src/operations/insert/mod.rs +++ b/grovedb/src/operations/insert/mod.rs @@ -852,6 +852,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 2, + sinsemilla_hash_calls: 0, } ); } @@ -919,6 +920,7 @@ mod tests { }, storage_loaded_bytes: 156, hash_node_calls: 8, + sinsemilla_hash_calls: 0, } ); } @@ -1002,6 +1004,7 @@ mod tests { }, storage_loaded_bytes: 232, hash_node_calls: 10, + sinsemilla_hash_calls: 0, } ); } @@ -1081,6 +1084,7 @@ mod tests { }, storage_loaded_bytes: 237, hash_node_calls: 10, + sinsemilla_hash_calls: 0, } ); } @@ -1142,6 +1146,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 2, + sinsemilla_hash_calls: 0, } ); } @@ -1202,6 +1207,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 3, // todo: verify this + sinsemilla_hash_calls: 0, } ); } @@ -1263,6 +1269,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 3, // todo: verify this + sinsemilla_hash_calls: 0, } ); } @@ -1327,6 +1334,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 3, + sinsemilla_hash_calls: 0, } ); } @@ -1407,6 +1415,7 @@ mod tests { }, storage_loaded_bytes: 152, // todo: verify this hash_node_calls: 8, // todo: verify this + sinsemilla_hash_calls: 0, } ); } @@ -1488,6 +1497,7 @@ mod tests { }, storage_loaded_bytes: 162, // todo: verify this hash_node_calls: 8, // todo: verify this + sinsemilla_hash_calls: 0, } ); } @@ -1553,6 +1563,7 @@ mod tests { }, storage_loaded_bytes: 0, hash_node_calls: 2, + sinsemilla_hash_calls: 0, } ); } @@ -1647,6 +1658,7 @@ mod tests { }, storage_loaded_bytes: 152, // todo: verify this hash_node_calls: 8, + sinsemilla_hash_calls: 0, } ); } @@ -1743,6 +1755,7 @@ mod tests { }, storage_loaded_bytes: 160, // todo: verify this hash_node_calls: 8, + sinsemilla_hash_calls: 0, } ); } @@ -1812,6 +1825,7 @@ mod tests { }, storage_loaded_bytes: 77, hash_node_calls: 2, + sinsemilla_hash_calls: 0, } ); } @@ -1866,6 +1880,7 @@ mod tests { }, storage_loaded_bytes: 230, // todo verify this hash_node_calls: 8, + sinsemilla_hash_calls: 0, } ); } @@ -1920,6 +1935,7 @@ mod tests { }, storage_loaded_bytes: 266, // todo verify this hash_node_calls: 9, + sinsemilla_hash_calls: 0, } ); } @@ -1986,6 +2002,7 @@ mod tests { }, storage_loaded_bytes: 487, // todo verify this hash_node_calls: 11, + sinsemilla_hash_calls: 0, } ); } @@ -2040,6 +2057,7 @@ mod tests { }, storage_loaded_bytes: 276, // todo verify this hash_node_calls: 9, + sinsemilla_hash_calls: 0, } ); } @@ -2094,6 +2112,7 @@ mod tests { }, storage_loaded_bytes: 231, hash_node_calls: 8, + sinsemilla_hash_calls: 0, } ); } @@ -2182,6 +2201,7 @@ mod tests { }, storage_loaded_bytes: 227, hash_node_calls: 9, // todo: verify this + sinsemilla_hash_calls: 0, } ); } diff --git a/grovedb/src/tests/provable_count_tree_test.rs b/grovedb/src/tests/provable_count_tree_test.rs index c40cc7853..def75119b 100644 --- a/grovedb/src/tests/provable_count_tree_test.rs +++ b/grovedb/src/tests/provable_count_tree_test.rs @@ -571,7 +571,7 @@ mod tests { .expect("should get root hash"); // Verify original proof works - let (root, results) = GroveDb::verify_query_raw(&proof, &path_query, grove_version) + let (root, _results) = GroveDb::verify_query_raw(&proof, &path_query, grove_version) .expect("original should verify"); assert_eq!(root, expected_root); diff --git a/grovedb/src/tests/test_provable_count_fresh.rs b/grovedb/src/tests/test_provable_count_fresh.rs index 5d6980394..9c0d15141 100644 --- a/grovedb/src/tests/test_provable_count_fresh.rs +++ b/grovedb/src/tests/test_provable_count_fresh.rs @@ -2,8 +2,8 @@ use grovedb_merk::proofs::Query; use grovedb_version::version::GroveVersion; use crate::{ - tests::{make_test_grovedb, TempGroveDb}, - Element, Error, GroveDb, PathQuery, + tests::make_test_grovedb, + Element, GroveDb, PathQuery, }; #[test] diff --git a/merk/src/element/get.rs b/merk/src/element/get.rs index e88002409..eca02ea4f 100644 --- a/merk/src/element/get.rs +++ b/merk/src/element/get.rs @@ -640,6 +640,7 @@ mod tests { storage_cost: Default::default(), storage_loaded_bytes: 0, hash_node_calls: 0, + sinsemilla_hash_calls: 0, } ); @@ -650,6 +651,7 @@ mod tests { storage_cost: Default::default(), storage_loaded_bytes: 75, hash_node_calls: 0, + sinsemilla_hash_calls: 0, } ); } From 5d227a1308325bf6e8ffb7beec3b3744498919ba Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 26 Feb 2026 17:33:19 +0700 Subject: [PATCH 3/6] more work --- .../src/client/sqlite_client_tests.rs | 13 +++++++------ .../src/client/sqlite_store/mod.rs | 1 - .../src/client/sqlite_store/tree_serialization.rs | 3 ++- .../src/client/sqlite_store_tests.rs | 6 ++---- grovedb/src/tests/test_provable_count_fresh.rs | 5 +---- 5 files changed, 12 insertions(+), 16 deletions(-) diff --git a/grovedb-commitment-tree/src/client/sqlite_client_tests.rs b/grovedb-commitment-tree/src/client/sqlite_client_tests.rs index c74041905..b7f00e379 100644 --- a/grovedb-commitment-tree/src/client/sqlite_client_tests.rs +++ b/grovedb-commitment-tree/src/client/sqlite_client_tests.rs @@ -131,13 +131,14 @@ mod tests { // Verify app table is still readable after commitment tree writes let guard = arc.lock().expect("lock"); let value: String = guard - .query_row( - "SELECT value FROM my_app_data WHERE id = 1", - [], - |row| row.get(0), - ) + .query_row("SELECT value FROM my_app_data WHERE id = 1", [], |row| { + row.get(0) + }) .expect("query app data"); - assert_eq!(value, "hello", "app data should survive commitment tree writes"); + assert_eq!( + value, "hello", + "app data should survive commitment tree writes" + ); } #[test] diff --git a/grovedb-commitment-tree/src/client/sqlite_store/mod.rs b/grovedb-commitment-tree/src/client/sqlite_store/mod.rs index abeb4b55d..ab9720cc3 100644 --- a/grovedb-commitment-tree/src/client/sqlite_store/mod.rs +++ b/grovedb-commitment-tree/src/client/sqlite_store/mod.rs @@ -25,7 +25,6 @@ use shardtree::{ store::{Checkpoint, ShardStore}, LocatedPrunableTree, PrunableTree, }; - use sql_helpers::*; // Re-export SHARD_HEIGHT from parent so sql_helpers can use it. diff --git a/grovedb-commitment-tree/src/client/sqlite_store/tree_serialization.rs b/grovedb-commitment-tree/src/client/sqlite_store/tree_serialization.rs index 8f333c9d5..2e2c778af 100644 --- a/grovedb-commitment-tree/src/client/sqlite_store/tree_serialization.rs +++ b/grovedb-commitment-tree/src/client/sqlite_store/tree_serialization.rs @@ -68,7 +68,8 @@ pub(crate) fn deserialize_tree( deserialize_tree_bounded(data, pos, 0) } -/// Depth-bounded deserialization to prevent stack overflow from malicious input. +/// Depth-bounded deserialization to prevent stack overflow from malicious +/// input. fn deserialize_tree_bounded( data: &[u8], pos: &mut usize, diff --git a/grovedb-commitment-tree/src/client/sqlite_store_tests.rs b/grovedb-commitment-tree/src/client/sqlite_store_tests.rs index 9ffade1d0..10cf0ac0a 100644 --- a/grovedb-commitment-tree/src/client/sqlite_store_tests.rs +++ b/grovedb-commitment-tree/src/client/sqlite_store_tests.rs @@ -50,10 +50,8 @@ mod tests { fn test_schema_idempotent() { let conn = Connection::open_in_memory().expect("open in-memory sqlite"); let arc = Arc::new(Mutex::new(conn)); - let _store1 = - SqliteShardStore::new_shared(arc.clone()).expect("first create"); - let _store2 = - SqliteShardStore::new_shared(arc.clone()).expect("second create on same DB"); + let _store1 = SqliteShardStore::new_shared(arc.clone()).expect("first create"); + let _store2 = SqliteShardStore::new_shared(arc.clone()).expect("second create on same DB"); } #[test] diff --git a/grovedb/src/tests/test_provable_count_fresh.rs b/grovedb/src/tests/test_provable_count_fresh.rs index 9c0d15141..5f50669d7 100644 --- a/grovedb/src/tests/test_provable_count_fresh.rs +++ b/grovedb/src/tests/test_provable_count_fresh.rs @@ -1,10 +1,7 @@ use grovedb_merk::proofs::Query; use grovedb_version::version::GroveVersion; -use crate::{ - tests::make_test_grovedb, - Element, GroveDb, PathQuery, -}; +use crate::{tests::make_test_grovedb, Element, GroveDb, PathQuery}; #[test] fn test_provable_count_tree_fresh_proof() { From a81261ae9d2c30a810a1a00aa4e133677d467299 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 26 Feb 2026 17:58:33 +0700 Subject: [PATCH 4/6] fix: eliminate test_leaf collision and reject invalid annotation flags - test_leaf now chains combine over all 8 bytes of the index instead of using index % 31, which caused collisions (e.g. 0 and 31) - Tree deserializer now explicitly matches 0x00/0x01 for the parent annotation flag and returns an error for any other value Co-Authored-By: Claude Opus 4.6 --- .../client/sqlite_store/tree_serialization.rs | 40 +++++++++++-------- grovedb-commitment-tree/src/test_utils.rs | 10 +++-- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/grovedb-commitment-tree/src/client/sqlite_store/tree_serialization.rs b/grovedb-commitment-tree/src/client/sqlite_store/tree_serialization.rs index 2e2c778af..4300494ab 100644 --- a/grovedb-commitment-tree/src/client/sqlite_store/tree_serialization.rs +++ b/grovedb-commitment-tree/src/client/sqlite_store/tree_serialization.rs @@ -123,24 +123,30 @@ fn deserialize_tree_bounded( let has_ann = data[*pos]; *pos += 1; - let ann: Option> = if has_ann == 0x01 { - if *pos + 32 > data.len() { - return Err(SqliteShardStoreError::Serialization( - "truncated parent annotation".to_string(), - )); + let ann: Option> = match has_ann { + 0x00 => None, + 0x01 => { + if *pos + 32 > data.len() { + return Err(SqliteShardStoreError::Serialization( + "truncated parent annotation".to_string(), + )); + } + let ann_bytes: [u8; 32] = data[*pos..*pos + 32] + .try_into() + .map_err(|_| SqliteShardStoreError::Serialization("bad ann".to_string()))?; + *pos += 32; + let hash = merkle_hash_from_bytes(&ann_bytes).ok_or_else(|| { + SqliteShardStoreError::Serialization( + "invalid Pallas field element in annotation".to_string(), + ) + })?; + Some(Arc::new(hash)) + } + other => { + return Err(SqliteShardStoreError::Serialization(format!( + "invalid parent annotation flag: 0x{other:02x}" + ))); } - let ann_bytes: [u8; 32] = data[*pos..*pos + 32] - .try_into() - .map_err(|_| SqliteShardStoreError::Serialization("bad ann".to_string()))?; - *pos += 32; - let hash = merkle_hash_from_bytes(&ann_bytes).ok_or_else(|| { - SqliteShardStoreError::Serialization( - "invalid Pallas field element in annotation".to_string(), - ) - })?; - Some(Arc::new(hash)) - } else { - None }; let left = deserialize_tree_bounded(data, pos, depth + 1)?; diff --git a/grovedb-commitment-tree/src/test_utils.rs b/grovedb-commitment-tree/src/test_utils.rs index f5c9ce1ec..681fcd3d2 100644 --- a/grovedb-commitment-tree/src/test_utils.rs +++ b/grovedb-commitment-tree/src/test_utils.rs @@ -6,9 +6,13 @@ use orchard::tree::MerkleHashOrchard; /// Create a deterministic test leaf from an index. /// /// Produces a valid Pallas field element (32 bytes) that is unique per index. -/// Uses Sinsemilla `combine` at different levels to produce varied hashes. +/// Chains Sinsemilla `combine` calls, one per byte of the index, so the full +/// 64-bit entropy is mixed in and no two distinct indices collide. pub fn test_leaf(index: u64) -> [u8; 32] { let empty = MerkleHashOrchard::empty_leaf(); - let varied = MerkleHashOrchard::combine(Level::from((index % 31) as u8 + 1), &empty, &empty); - MerkleHashOrchard::combine(Level::from(0), &empty, &varied).to_bytes() + let mut current = empty; + for &byte in index.to_le_bytes().iter() { + current = MerkleHashOrchard::combine(Level::from(byte), ¤t, &empty); + } + current.to_bytes() } From de33d1388c1014f9cfd3dbdb2effdc5de9725c54 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 26 Feb 2026 18:10:05 +0700 Subject: [PATCH 5/6] chore: fix clippy warnings and enforce missing_docs lint - Use += for OperationCost addition (assign_op_pattern) - Remove module_inception: unwrap inner mod tests from tests.rs files - Replace expect(&format!()) with unwrap_or_else (expect_fun_call) - Add #![warn(missing_docs)] to crate root - Add doc comments to all CommitmentTreeError variants and fields Co-Authored-By: Claude Opus 4.6 --- grovedb-commitment-tree/src/client/tests.rs | 445 +++++++------- .../src/commitment_frontier/mod.rs | 1 + .../src/commitment_frontier/tests.rs | 554 +++++++++--------- .../src/commitment_tree/mod.rs | 4 +- grovedb-commitment-tree/src/error.rs | 12 +- grovedb-commitment-tree/src/lib.rs | 1 + 6 files changed, 511 insertions(+), 506 deletions(-) diff --git a/grovedb-commitment-tree/src/client/tests.rs b/grovedb-commitment-tree/src/client/tests.rs index f3b04e23e..361392cbe 100644 --- a/grovedb-commitment-tree/src/client/tests.rs +++ b/grovedb-commitment-tree/src/client/tests.rs @@ -1,249 +1,246 @@ -#[cfg(test)] -mod tests { - use incrementalmerkletree::{Position, Retention}; - use orchard::tree::Anchor; - - use crate::{test_utils::test_leaf, ClientMemoryCommitmentTree}; - - #[test] - fn test_empty_tree() { - let tree = ClientMemoryCommitmentTree::new(10); - assert_eq!(tree.max_leaf_position().expect("max_leaf_position"), None); - assert_eq!(tree.anchor().expect("anchor"), Anchor::empty_tree()); - } +use incrementalmerkletree::{Position, Retention}; +use orchard::tree::Anchor; - #[test] - fn test_append_and_position() { - let mut tree = ClientMemoryCommitmentTree::new(10); +use crate::{test_utils::test_leaf, ClientMemoryCommitmentTree}; - tree.append(test_leaf(0), Retention::Marked) - .expect("append 0"); - assert_eq!( - tree.max_leaf_position().expect("max_leaf_position"), - Some(Position::from(0)) - ); +#[test] +fn test_empty_tree() { + let tree = ClientMemoryCommitmentTree::new(10); + assert_eq!(tree.max_leaf_position().expect("max_leaf_position"), None); + assert_eq!(tree.anchor().expect("anchor"), Anchor::empty_tree()); +} - tree.append(test_leaf(1), Retention::Ephemeral) - .expect("append 1"); - assert_eq!( - tree.max_leaf_position().expect("max_leaf_position"), - Some(Position::from(1)) - ); - } +#[test] +fn test_append_and_position() { + let mut tree = ClientMemoryCommitmentTree::new(10); + + tree.append(test_leaf(0), Retention::Marked) + .expect("append 0"); + assert_eq!( + tree.max_leaf_position().expect("max_leaf_position"), + Some(Position::from(0)) + ); + + tree.append(test_leaf(1), Retention::Ephemeral) + .expect("append 1"); + assert_eq!( + tree.max_leaf_position().expect("max_leaf_position"), + Some(Position::from(1)) + ); +} - #[test] - fn test_anchor_changes() { - let mut tree = ClientMemoryCommitmentTree::new(10); - let empty_anchor = tree.anchor().expect("anchor"); +#[test] +fn test_anchor_changes() { + let mut tree = ClientMemoryCommitmentTree::new(10); + let empty_anchor = tree.anchor().expect("anchor"); - tree.append(test_leaf(0), Retention::Marked) - .expect("append 0"); - let anchor1 = tree.anchor().expect("anchor"); - assert_ne!(empty_anchor, anchor1); + tree.append(test_leaf(0), Retention::Marked) + .expect("append 0"); + let anchor1 = tree.anchor().expect("anchor"); + assert_ne!(empty_anchor, anchor1); - tree.append(test_leaf(1), Retention::Marked) - .expect("append 1"); - let anchor2 = tree.anchor().expect("anchor"); - assert_ne!(anchor1, anchor2); - } + tree.append(test_leaf(1), Retention::Marked) + .expect("append 1"); + let anchor2 = tree.anchor().expect("anchor"); + assert_ne!(anchor1, anchor2); +} + +#[test] +fn test_witness_generation() { + let mut tree = ClientMemoryCommitmentTree::new(10); - #[test] - fn test_witness_generation() { - let mut tree = ClientMemoryCommitmentTree::new(10); + // Append a marked leaf so we can witness it + tree.append(test_leaf(0), Retention::Marked) + .expect("append 0"); + tree.append(test_leaf(1), Retention::Ephemeral) + .expect("append 1"); + tree.checkpoint(1).expect("checkpoint"); - // Append a marked leaf so we can witness it - tree.append(test_leaf(0), Retention::Marked) - .expect("append 0"); - tree.append(test_leaf(1), Retention::Ephemeral) - .expect("append 1"); - tree.checkpoint(1).expect("checkpoint"); + // Witness for position 0 at current state + let path = tree.witness(Position::from(0), 0).expect("witness"); + assert!(path.is_some(), "should produce witness for marked leaf"); +} - // Witness for position 0 at current state - let path = tree.witness(Position::from(0), 0).expect("witness"); - assert!(path.is_some(), "should produce witness for marked leaf"); +#[test] +#[cfg(feature = "server")] +fn test_frontier_and_client_same_root() { + use crate::commitment_frontier::CommitmentFrontier; + + let mut frontier = CommitmentFrontier::new(); + let mut client = ClientMemoryCommitmentTree::new(10); + + for i in 0..20u64 { + frontier + .append(test_leaf(i)) + .value + .expect("frontier append"); + client + .append(test_leaf(i), Retention::Ephemeral) + .expect("client append"); } - #[test] - #[cfg(feature = "server")] - fn test_frontier_and_client_same_root() { - use crate::commitment_frontier::CommitmentFrontier; - - let mut frontier = CommitmentFrontier::new(); - let mut client = ClientMemoryCommitmentTree::new(10); - - for i in 0..20u64 { - frontier - .append(test_leaf(i)) - .value - .expect("frontier append"); - client - .append(test_leaf(i), Retention::Ephemeral) - .expect("client append"); - } - - assert_eq!(frontier.anchor(), client.anchor().expect("client anchor")); + assert_eq!(frontier.anchor(), client.anchor().expect("client anchor")); +} + +/// Demonstrates that `checkpoint()` with a duplicate ID silently returns +/// `Ok(false)` and does NOT advance the checkpoint frontier. Notes +/// appended after the original checkpoint are unreachable by +/// `witness_at_checkpoint_depth(pos, 0)`. +/// +/// This is the exact failure mode that caused the "Tree does not contain +/// a root at address" error in PMT when the sync code reused +/// `next_start_index` as the checkpoint ID across re-syncs. +#[test] +fn test_duplicate_checkpoint_id_breaks_witness_for_new_notes() { + let mut tree = ClientMemoryCommitmentTree::new(100); + + // Sync 1: append 20 notes (even = Marked, odd = Ephemeral) + for i in 0..20u64 { + let retention = if i % 2 == 0 { + Retention::Marked + } else { + Retention::Ephemeral + }; + tree.append(test_leaf(i), retention).expect("append sync 1"); } - /// Demonstrates that `checkpoint()` with a duplicate ID silently returns - /// `Ok(false)` and does NOT advance the checkpoint frontier. Notes - /// appended after the original checkpoint are unreachable by - /// `witness_at_checkpoint_depth(pos, 0)`. - /// - /// This is the exact failure mode that caused the "Tree does not contain - /// a root at address" error in PMT when the sync code reused - /// `next_start_index` as the checkpoint ID across re-syncs. - #[test] - fn test_duplicate_checkpoint_id_breaks_witness_for_new_notes() { - let mut tree = ClientMemoryCommitmentTree::new(100); - - // Sync 1: append 20 notes (even = Marked, odd = Ephemeral) - for i in 0..20u64 { - let retention = if i % 2 == 0 { - Retention::Marked - } else { - Retention::Ephemeral - }; - tree.append(test_leaf(i), retention).expect("append sync 1"); - } - - // Checkpoint with the "chunk boundary" ID - let created = tree.checkpoint(2048).expect("checkpoint 1"); - assert!(created, "first checkpoint should succeed"); - - // Witness works for all marked notes in sync 1 - for i in (0..20u64).step_by(2) { - let path = tree - .witness(Position::from(i), 0) - .expect("witness sync 1 note"); - assert!( - path.is_some(), - "should produce witness for marked note at position {}", - i - ); - } - - // Sync 2: append 30 more notes (simulates new notes arriving) - for i in 20..50u64 { - let retention = if i % 2 == 0 { - Retention::Marked - } else { - Retention::Ephemeral - }; - tree.append(test_leaf(i), retention).expect("append sync 2"); - } - - // BUG: reuse the same checkpoint ID — returns Ok(false)! - let created = tree.checkpoint(2048).expect("checkpoint 2 (duplicate)"); - assert!( - !created, - "duplicate checkpoint ID should return false (no new checkpoint created)" - ); + // Checkpoint with the "chunk boundary" ID + let created = tree.checkpoint(2048).expect("checkpoint 1"); + assert!(created, "first checkpoint should succeed"); - // Original sync 1 notes still have valid witnesses + // Witness works for all marked notes in sync 1 + for i in (0..20u64).step_by(2) { let path = tree - .witness(Position::from(0), 0) - .expect("witness sync 1 note after sync 2"); - assert!(path.is_some(), "sync 1 notes should still be witnessable"); - - // Sync 2 notes at positions >= 20 CANNOT be witnessed because the - // checkpoint is stuck at position 19 (from sync 1). This is the bug. - let result = tree.witness(Position::from(20), 0); + .witness(Position::from(i), 0) + .expect("witness sync 1 note"); assert!( - result.is_err(), - "witness should fail for notes beyond the stale checkpoint" + path.is_some(), + "should produce witness for marked note at position {}", + i ); } - /// Shows the correct pattern: use unique, increasing checkpoint IDs - /// so that each sync creates a new checkpoint covering all appended notes. - #[test] - fn test_unique_checkpoint_ids_allow_witness_for_all_notes() { - let mut tree = ClientMemoryCommitmentTree::new(100); - - // Sync 1: append 20 notes - for i in 0..20u64 { - let retention = if i % 2 == 0 { - Retention::Marked - } else { - Retention::Ephemeral - }; - tree.append(test_leaf(i), retention).expect("append sync 1"); - } - - // Checkpoint with unique ID = last appended position - let created = tree.checkpoint(19).expect("checkpoint 1"); - assert!(created, "first checkpoint should succeed"); - - // Sync 2: append 30 more notes - for i in 20..50u64 { - let retention = if i % 2 == 0 { - Retention::Marked - } else { - Retention::Ephemeral - }; - tree.append(test_leaf(i), retention).expect("append sync 2"); - } - - // Checkpoint with new unique ID = new last appended position - let created = tree.checkpoint(49).expect("checkpoint 2"); - assert!(created, "second checkpoint with unique ID should succeed"); - - // ALL marked notes — from both syncs — can be witnessed - for i in (0..50u64).step_by(2) { - let path = tree - .witness(Position::from(i), 0) - .expect(&format!("witness note at position {}", i)); - assert!( - path.is_some(), - "should produce witness for marked note at position {}", - i - ); - } + // Sync 2: append 30 more notes (simulates new notes arriving) + for i in 20..50u64 { + let retention = if i % 2 == 0 { + Retention::Marked + } else { + Retention::Ephemeral + }; + tree.append(test_leaf(i), retention).expect("append sync 2"); + } + + // BUG: reuse the same checkpoint ID — returns Ok(false)! + let created = tree.checkpoint(2048).expect("checkpoint 2 (duplicate)"); + assert!( + !created, + "duplicate checkpoint ID should return false (no new checkpoint created)" + ); + + // Original sync 1 notes still have valid witnesses + let path = tree + .witness(Position::from(0), 0) + .expect("witness sync 1 note after sync 2"); + assert!(path.is_some(), "sync 1 notes should still be witnessable"); + + // Sync 2 notes at positions >= 20 CANNOT be witnessed because the + // checkpoint is stuck at position 19 (from sync 1). This is the bug. + let result = tree.witness(Position::from(20), 0); + assert!( + result.is_err(), + "witness should fail for notes beyond the stale checkpoint" + ); +} + +/// Shows the correct pattern: use unique, increasing checkpoint IDs +/// so that each sync creates a new checkpoint covering all appended notes. +#[test] +fn test_unique_checkpoint_ids_allow_witness_for_all_notes() { + let mut tree = ClientMemoryCommitmentTree::new(100); + + // Sync 1: append 20 notes + for i in 0..20u64 { + let retention = if i % 2 == 0 { + Retention::Marked + } else { + Retention::Ephemeral + }; + tree.append(test_leaf(i), retention).expect("append sync 1"); + } + + // Checkpoint with unique ID = last appended position + let created = tree.checkpoint(19).expect("checkpoint 1"); + assert!(created, "first checkpoint should succeed"); + + // Sync 2: append 30 more notes + for i in 20..50u64 { + let retention = if i % 2 == 0 { + Retention::Marked + } else { + Retention::Ephemeral + }; + tree.append(test_leaf(i), retention).expect("append sync 2"); } - /// Verifies that witness anchors from both syncs match when using - /// unique checkpoint IDs, and that the anchor at checkpoint depth 1 - /// differs from depth 0 (since the tree grew between checkpoints). - #[test] - fn test_witness_anchors_match_across_syncs() { - let mut tree = ClientMemoryCommitmentTree::new(100); - - // Sync 1 - for i in 0..10u64 { - tree.append(test_leaf(i), Retention::Marked) - .expect("append sync 1"); - } - tree.checkpoint(9).expect("checkpoint 1"); - let anchor_after_sync1 = tree.anchor().expect("anchor after sync 1"); - - // Sync 2 - for i in 10..20u64 { - tree.append(test_leaf(i), Retention::Marked) - .expect("append sync 2"); - } - tree.checkpoint(19).expect("checkpoint 2"); - let anchor_after_sync2 = tree.anchor().expect("anchor after sync 2"); - - // Anchors should differ (tree grew) - assert_ne!( - anchor_after_sync1, anchor_after_sync2, - "anchors should differ after tree growth" + // Checkpoint with new unique ID = new last appended position + let created = tree.checkpoint(49).expect("checkpoint 2"); + assert!(created, "second checkpoint with unique ID should succeed"); + + // ALL marked notes — from both syncs — can be witnessed + for i in (0..50u64).step_by(2) { + let path = tree + .witness(Position::from(i), 0) + .unwrap_or_else(|_| panic!("witness note at position {}", i)); + assert!( + path.is_some(), + "should produce witness for marked note at position {}", + i ); + } +} + +/// Verifies that witness anchors from both syncs match when using +/// unique checkpoint IDs, and that the anchor at checkpoint depth 1 +/// differs from depth 0 (since the tree grew between checkpoints). +#[test] +fn test_witness_anchors_match_across_syncs() { + let mut tree = ClientMemoryCommitmentTree::new(100); + + // Sync 1 + for i in 0..10u64 { + tree.append(test_leaf(i), Retention::Marked) + .expect("append sync 1"); + } + tree.checkpoint(9).expect("checkpoint 1"); + let anchor_after_sync1 = tree.anchor().expect("anchor after sync 1"); - // Witness at depth 0 uses the latest checkpoint (sync 2) - let path_depth0 = tree - .witness(Position::from(0), 0) - .expect("witness at depth 0"); - assert!(path_depth0.is_some()); - - // Witness at depth 1 uses the previous checkpoint (sync 1) - let path_depth1 = tree - .witness(Position::from(0), 1) - .expect("witness at depth 1"); - assert!(path_depth1.is_some()); - - // Both witnesses exist at their respective checkpoint depths - // (MerklePath doesn't implement PartialEq so we just verify both are - // Some) + // Sync 2 + for i in 10..20u64 { + tree.append(test_leaf(i), Retention::Marked) + .expect("append sync 2"); } + tree.checkpoint(19).expect("checkpoint 2"); + let anchor_after_sync2 = tree.anchor().expect("anchor after sync 2"); + + // Anchors should differ (tree grew) + assert_ne!( + anchor_after_sync1, anchor_after_sync2, + "anchors should differ after tree growth" + ); + + // Witness at depth 0 uses the latest checkpoint (sync 2) + let path_depth0 = tree + .witness(Position::from(0), 0) + .expect("witness at depth 0"); + assert!(path_depth0.is_some()); + + // Witness at depth 1 uses the previous checkpoint (sync 1) + let path_depth1 = tree + .witness(Position::from(0), 1) + .expect("witness at depth 1"); + assert!(path_depth1.is_some()); + + // Both witnesses exist at their respective checkpoint depths + // (MerklePath doesn't implement PartialEq so we just verify both are + // Some) } diff --git a/grovedb-commitment-tree/src/commitment_frontier/mod.rs b/grovedb-commitment-tree/src/commitment_frontier/mod.rs index 36f461e87..fae356304 100644 --- a/grovedb-commitment-tree/src/commitment_frontier/mod.rs +++ b/grovedb-commitment-tree/src/commitment_frontier/mod.rs @@ -4,6 +4,7 @@ use orchard::{tree::MerkleHashOrchard, Anchor, NOTE_COMMITMENT_TREE_DEPTH}; pub use crate::error::CommitmentTreeError; +#[cfg(all(test, feature = "server"))] mod tests; /// Depth of the Sinsemilla Merkle tree as a u8 constant for the Frontier type diff --git a/grovedb-commitment-tree/src/commitment_frontier/tests.rs b/grovedb-commitment-tree/src/commitment_frontier/tests.rs index d2b97acca..75ae28bb7 100644 --- a/grovedb-commitment-tree/src/commitment_frontier/tests.rs +++ b/grovedb-commitment-tree/src/commitment_frontier/tests.rs @@ -1,312 +1,308 @@ -#[cfg(all(test, feature = "server"))] -mod tests { - use incrementalmerkletree::{Hashable, Level}; - use orchard::{ - tree::{Anchor, MerkleHashOrchard}, - NOTE_COMMITMENT_TREE_DEPTH, - }; - - use crate::{ - commitment_frontier::{empty_sinsemilla_root, CommitmentFrontier, EMPTY_SINSEMILLA_ROOT}, - test_utils::test_leaf, - }; - - #[test] - fn test_empty_frontier() { - let f = CommitmentFrontier::new(); - assert_eq!(f.position(), None); - assert_eq!(f.tree_size(), 0); - - let empty_anchor = Anchor::empty_tree(); - assert_eq!(f.anchor(), empty_anchor); - } +use incrementalmerkletree::{Hashable, Level}; +use orchard::{ + tree::{Anchor, MerkleHashOrchard}, + NOTE_COMMITMENT_TREE_DEPTH, +}; + +use crate::{ + commitment_frontier::{empty_sinsemilla_root, CommitmentFrontier, EMPTY_SINSEMILLA_ROOT}, + test_utils::test_leaf, +}; + +#[test] +fn test_empty_frontier() { + let f = CommitmentFrontier::new(); + assert_eq!(f.position(), None); + assert_eq!(f.tree_size(), 0); + + let empty_anchor = Anchor::empty_tree(); + assert_eq!(f.anchor(), empty_anchor); +} - #[test] - fn test_append_changes_root() { - let mut f = CommitmentFrontier::new(); - let empty_root = f.root_hash(); +#[test] +fn test_append_changes_root() { + let mut f = CommitmentFrontier::new(); + let empty_root = f.root_hash(); - let result = f.append(test_leaf(0)); - let new_root = result.value.expect("append should succeed"); - assert_ne!(empty_root, new_root); - assert_eq!(f.root_hash(), new_root); - } + let result = f.append(test_leaf(0)); + let new_root = result.value.expect("append should succeed"); + assert_ne!(empty_root, new_root); + assert_eq!(f.root_hash(), new_root); +} - #[test] - fn test_append_tracks_position() { - let mut f = CommitmentFrontier::new(); - assert_eq!(f.position(), None); - assert_eq!(f.tree_size(), 0); - - f.append(test_leaf(0)).value.expect("append 0"); - assert_eq!(f.position(), Some(0)); - assert_eq!(f.tree_size(), 1); - - f.append(test_leaf(1)).value.expect("append 1"); - assert_eq!(f.position(), Some(1)); - assert_eq!(f.tree_size(), 2); - - for i in 2..100u64 { - f.append(test_leaf(i)).value.expect("append loop"); - } - assert_eq!(f.position(), Some(99)); - assert_eq!(f.tree_size(), 100); - } +#[test] +fn test_append_tracks_position() { + let mut f = CommitmentFrontier::new(); + assert_eq!(f.position(), None); + assert_eq!(f.tree_size(), 0); - #[test] - fn test_deterministic_roots() { - let mut f1 = CommitmentFrontier::new(); - let mut f2 = CommitmentFrontier::new(); + f.append(test_leaf(0)).value.expect("append 0"); + assert_eq!(f.position(), Some(0)); + assert_eq!(f.tree_size(), 1); - for i in 0..10u64 { - f1.append(test_leaf(i)).value.expect("append f1"); - f2.append(test_leaf(i)).value.expect("append f2"); - } + f.append(test_leaf(1)).value.expect("append 1"); + assert_eq!(f.position(), Some(1)); + assert_eq!(f.tree_size(), 2); - assert_eq!(f1.root_hash(), f2.root_hash()); + for i in 2..100u64 { + f.append(test_leaf(i)).value.expect("append loop"); } + assert_eq!(f.position(), Some(99)); + assert_eq!(f.tree_size(), 100); +} - #[test] - fn test_different_leaves_different_roots() { - let mut f1 = CommitmentFrontier::new(); - let mut f2 = CommitmentFrontier::new(); - - f1.append(test_leaf(0)).value.expect("append f1"); - f2.append(test_leaf(1)).value.expect("append f2"); +#[test] +fn test_deterministic_roots() { + let mut f1 = CommitmentFrontier::new(); + let mut f2 = CommitmentFrontier::new(); - assert_ne!(f1.root_hash(), f2.root_hash()); + for i in 0..10u64 { + f1.append(test_leaf(i)).value.expect("append f1"); + f2.append(test_leaf(i)).value.expect("append f2"); } - #[test] - fn test_serialize_empty() { - let f = CommitmentFrontier::new(); - let data = f.serialize(); - let f2 = CommitmentFrontier::deserialize(&data).expect("deserialize empty frontier"); + assert_eq!(f1.root_hash(), f2.root_hash()); +} - assert_eq!(f.root_hash(), f2.root_hash()); - assert_eq!(f.position(), f2.position()); - } +#[test] +fn test_different_leaves_different_roots() { + let mut f1 = CommitmentFrontier::new(); + let mut f2 = CommitmentFrontier::new(); - #[test] - fn test_serialize_roundtrip() { - let mut f = CommitmentFrontier::new(); - for i in 0..100u64 { - f.append(test_leaf(i)).value.expect("append"); - } + f1.append(test_leaf(0)).value.expect("append f1"); + f2.append(test_leaf(1)).value.expect("append f2"); - let data = f.serialize(); - let f2 = CommitmentFrontier::deserialize(&data).expect("deserialize frontier"); + assert_ne!(f1.root_hash(), f2.root_hash()); +} - assert_eq!(f.root_hash(), f2.root_hash()); - assert_eq!(f.position(), f2.position()); - assert_eq!(f.tree_size(), f2.tree_size()); - } +#[test] +fn test_serialize_empty() { + let f = CommitmentFrontier::new(); + let data = f.serialize(); + let f2 = CommitmentFrontier::deserialize(&data).expect("deserialize empty frontier"); - #[test] - fn test_serialize_roundtrip_with_many_leaves() { - let mut f = CommitmentFrontier::new(); - for i in 0..1000u64 { - f.append(test_leaf(i)).value.expect("append"); - } - - let data = f.serialize(); - // Frontier should be small regardless of leaf count - // 1 (flag) + 8 (position) + 32 (leaf) + 1 (ommer_count) + N*32 (ommers) - // Max ommers for depth 32 = 32, so max ~1.1KB - assert!( - data.len() < 1200, - "frontier serialized to {} bytes", - data.len() - ); - - let f2 = - CommitmentFrontier::deserialize(&data).expect("deserialize frontier with many leaves"); - assert_eq!(f.root_hash(), f2.root_hash()); - assert_eq!(f.tree_size(), f2.tree_size()); - } + assert_eq!(f.root_hash(), f2.root_hash()); + assert_eq!(f.position(), f2.position()); +} - #[test] - fn test_invalid_field_element() { - // All 0xFF bytes is not a valid Pallas field element - let result = CommitmentFrontier::new().append([0xff; 32]); - assert!(result.value.is_err()); +#[test] +fn test_serialize_roundtrip() { + let mut f = CommitmentFrontier::new(); + for i in 0..100u64 { + f.append(test_leaf(i)).value.expect("append"); } - #[test] - fn test_deserialize_invalid_data() { - assert!(CommitmentFrontier::deserialize(&[]).is_err()); - assert!(CommitmentFrontier::deserialize(&[0x02]).is_err()); - assert!(CommitmentFrontier::deserialize(&[0x01]).is_err()); - } + let data = f.serialize(); + let f2 = CommitmentFrontier::deserialize(&data).expect("deserialize frontier"); - #[test] - fn test_root_hash_is_32_bytes() { - let f = CommitmentFrontier::new(); - assert_eq!(f.root_hash().len(), 32); - } + assert_eq!(f.root_hash(), f2.root_hash()); + assert_eq!(f.position(), f2.position()); + assert_eq!(f.tree_size(), f2.tree_size()); +} - #[test] - fn test_empty_tree_root_matches_orchard() { - let f = CommitmentFrontier::new(); - let root = f.root_hash(); - let expected = - MerkleHashOrchard::empty_root(Level::from(NOTE_COMMITMENT_TREE_DEPTH as u8)).to_bytes(); - assert_eq!(root, expected); +#[test] +fn test_serialize_roundtrip_with_many_leaves() { + let mut f = CommitmentFrontier::new(); + for i in 0..1000u64 { + f.append(test_leaf(i)).value.expect("append"); } - #[test] - fn test_empty_sinsemilla_root_constant() { - // Verify the precomputed constant matches the runtime value - let computed = empty_sinsemilla_root(); - assert_eq!( - computed, EMPTY_SINSEMILLA_ROOT, - "EMPTY_SINSEMILLA_ROOT constant is stale. Update it to: {:?}", - computed - ); - } + let data = f.serialize(); + // Frontier should be small regardless of leaf count + // 1 (flag) + 8 (position) + 32 (leaf) + 1 (ommer_count) + N*32 (ommers) + // Max ommers for depth 32 = 32, so max ~1.1KB + assert!( + data.len() < 1200, + "frontier serialized to {} bytes", + data.len() + ); + + let f2 = CommitmentFrontier::deserialize(&data).expect("deserialize frontier with many leaves"); + assert_eq!(f.root_hash(), f2.root_hash()); + assert_eq!(f.tree_size(), f2.tree_size()); +} - #[test] - fn test_default_impl() { - let f = CommitmentFrontier::default(); - assert_eq!(f.position(), None); - assert_eq!(f.tree_size(), 0); - assert_eq!(f.root_hash(), CommitmentFrontier::new().root_hash()); - } +#[test] +fn test_invalid_field_element() { + // All 0xFF bytes is not a valid Pallas field element + let result = CommitmentFrontier::new().append([0xff; 32]); + assert!(result.value.is_err()); +} - #[test] - fn test_deserialize_truncated_ommers() { - // Build a valid serialized frontier with 1 leaf so we know the ommer - // count byte, then truncate the ommer data. - let mut f = CommitmentFrontier::new(); - // Append enough leaves to generate ommers. After 3 appends (positions - // 0,1,2), position=2 has trailing_ones=0 so ommer_count may be 1. - // After 4 appends position=3 has trailing_ones=2, generating ommers. - for i in 0..4u64 { - f.append(test_leaf(i)).value.expect("append"); - } - let data = f.serialize(); - // data layout: 1 (flag) + 8 (position) + 32 (leaf) + 1 (ommer_count) + N*32 - let ommer_count = data[42] as usize; - assert!( - ommer_count > 0, - "need at least one ommer to test truncation" - ); - // Truncate: keep header + ommer_count byte but chop the ommer data - let truncated = &data[..43]; - let err = CommitmentFrontier::deserialize(truncated); - assert!(err.is_err(), "should fail on truncated ommers"); - let msg = format!("{}", err.expect_err("should be an error")); - assert!( - msg.contains("truncated ommers"), - "expected 'truncated ommers' error, got: {msg}" - ); - } +#[test] +fn test_deserialize_invalid_data() { + assert!(CommitmentFrontier::deserialize(&[]).is_err()); + assert!(CommitmentFrontier::deserialize(&[0x02]).is_err()); + assert!(CommitmentFrontier::deserialize(&[0x01]).is_err()); +} - #[test] - fn test_deserialize_invalid_leaf_field_element() { - // Construct bytes with valid header but an invalid Pallas field element - // as the leaf (all 0xFF is not a valid point). - let mut data = vec![0x01]; // has_frontier = true - data.extend_from_slice(&0u64.to_be_bytes()); // position = 0 - data.extend_from_slice(&[0xFF; 32]); // invalid leaf - data.push(0); // ommer_count = 0 - - let err = CommitmentFrontier::deserialize(&data); - assert!(err.is_err(), "should fail on invalid leaf field element"); - let msg = format!("{}", err.expect_err("should be an error")); - assert!( - msg.contains("invalid Pallas field element"), - "expected InvalidFieldElement error, got: {msg}" - ); - } +#[test] +fn test_root_hash_is_32_bytes() { + let f = CommitmentFrontier::new(); + assert_eq!(f.root_hash().len(), 32); +} - #[test] - fn test_deserialize_invalid_ommer_field_element() { - // Build a valid frontier, then replace one ommer with 0xFF bytes. - let mut f = CommitmentFrontier::new(); - for i in 0..4u64 { - f.append(test_leaf(i)).value.expect("append"); - } - let mut data = f.serialize(); - let ommer_count = data[42] as usize; - assert!(ommer_count > 0, "need at least one ommer"); - // First ommer starts at byte 43, replace it with all 0xFF - for b in &mut data[43..43 + 32] { - *b = 0xFF; - } - let err = CommitmentFrontier::deserialize(&data); - assert!(err.is_err(), "should fail on invalid ommer field element"); - let msg = format!("{}", err.expect_err("should be an error")); - assert!( - msg.contains("invalid Pallas field element"), - "expected InvalidFieldElement error, got: {msg}" - ); - } +#[test] +fn test_empty_tree_root_matches_orchard() { + let f = CommitmentFrontier::new(); + let root = f.root_hash(); + let expected = + MerkleHashOrchard::empty_root(Level::from(NOTE_COMMITMENT_TREE_DEPTH as u8)).to_bytes(); + assert_eq!(root, expected); +} - #[test] - fn test_deserialize_from_parts_failure() { - // Construct technically valid field elements but with an inconsistent - // position/ommer combination that `Frontier::from_parts` rejects. - // Position 0 should have 0 ommers; providing 1 ommer triggers the - // from_parts validation error. - let valid_leaf = test_leaf(0); - let valid_ommer = test_leaf(1); - - let mut data = vec![0x01]; // has_frontier - data.extend_from_slice(&0u64.to_be_bytes()); // position = 0 - data.extend_from_slice(&valid_leaf); // leaf - data.push(1); // ommer_count = 1 (wrong for position 0) - data.extend_from_slice(&valid_ommer); // ommer - - let err = CommitmentFrontier::deserialize(&data); - assert!(err.is_err(), "should fail on inconsistent from_parts"); - let msg = format!("{}", err.expect_err("should be an error")); - assert!( - msg.contains("frontier reconstruction"), - "expected 'frontier reconstruction' error, got: {msg}" - ); - } +#[test] +fn test_empty_sinsemilla_root_constant() { + // Verify the precomputed constant matches the runtime value + let computed = empty_sinsemilla_root(); + assert_eq!( + computed, EMPTY_SINSEMILLA_ROOT, + "EMPTY_SINSEMILLA_ROOT constant is stale. Update it to: {:?}", + computed + ); +} + +#[test] +fn test_default_impl() { + let f = CommitmentFrontier::default(); + assert_eq!(f.position(), None); + assert_eq!(f.tree_size(), 0); + assert_eq!(f.root_hash(), CommitmentFrontier::new().root_hash()); +} - #[test] - fn test_append_cost_sinsemilla_hash_calls() { - let mut f = CommitmentFrontier::new(); - - // First append (position 0): 32 hashes + 0 trailing_ones(empty) = 32 - let r0 = f.append(test_leaf(0)); - r0.value.expect("append 0"); - assert_eq!(r0.cost.sinsemilla_hash_calls, 32); - - // Second append (position 0 in frontier before append): trailing_ones(0) - // = 0 0 in binary is ...0, trailing_ones = 0, so 32 + 0 = 32 - let r1 = f.append(test_leaf(1)); - r1.value.expect("append 1"); - assert_eq!(r1.cost.sinsemilla_hash_calls, 32); - - // Third append (position 1): trailing_ones(1) = 1, so 32 + 1 = 33 - let r2 = f.append(test_leaf(2)); - r2.value.expect("append 2"); - assert_eq!(r2.cost.sinsemilla_hash_calls, 33); - - // Fourth append (position 2): trailing_ones(2=0b10) = 0, so 32 - let r3 = f.append(test_leaf(3)); - r3.value.expect("append 3"); - assert_eq!(r3.cost.sinsemilla_hash_calls, 32); - - // Fifth append (position 3): trailing_ones(3=0b11) = 2, so 34 - let r4 = f.append(test_leaf(4)); - r4.value.expect("append 4"); - assert_eq!(r4.cost.sinsemilla_hash_calls, 34); +#[test] +fn test_deserialize_truncated_ommers() { + // Build a valid serialized frontier with 1 leaf so we know the ommer + // count byte, then truncate the ommer data. + let mut f = CommitmentFrontier::new(); + // Append enough leaves to generate ommers. After 3 appends (positions + // 0,1,2), position=2 has trailing_ones=0 so ommer_count may be 1. + // After 4 appends position=3 has trailing_ones=2, generating ommers. + for i in 0..4u64 { + f.append(test_leaf(i)).value.expect("append"); } + let data = f.serialize(); + // data layout: 1 (flag) + 8 (position) + 32 (leaf) + 1 (ommer_count) + N*32 + let ommer_count = data[42] as usize; + assert!( + ommer_count > 0, + "need at least one ommer to test truncation" + ); + // Truncate: keep header + ommer_count byte but chop the ommer data + let truncated = &data[..43]; + let err = CommitmentFrontier::deserialize(truncated); + assert!(err.is_err(), "should fail on truncated ommers"); + let msg = format!("{}", err.expect_err("should be an error")); + assert!( + msg.contains("truncated ommers"), + "expected 'truncated ommers' error, got: {msg}" + ); +} + +#[test] +fn test_deserialize_invalid_leaf_field_element() { + // Construct bytes with valid header but an invalid Pallas field element + // as the leaf (all 0xFF is not a valid point). + let mut data = vec![0x01]; // has_frontier = true + data.extend_from_slice(&0u64.to_be_bytes()); // position = 0 + data.extend_from_slice(&[0xFF; 32]); // invalid leaf + data.push(0); // ommer_count = 0 + + let err = CommitmentFrontier::deserialize(&data); + assert!(err.is_err(), "should fail on invalid leaf field element"); + let msg = format!("{}", err.expect_err("should be an error")); + assert!( + msg.contains("invalid Pallas field element"), + "expected InvalidFieldElement error, got: {msg}" + ); +} - #[test] - fn test_deserialize_invalid_frontier_flag() { - // Test with a frontier flag value that is neither 0x00 nor 0x01 - let err = CommitmentFrontier::deserialize(&[0x42]); - assert!(err.is_err()); - let msg = format!("{}", err.expect_err("should be an error")); - assert!( - msg.contains("invalid frontier flag: 0x42"), - "expected 'invalid frontier flag' error, got: {msg}" - ); +#[test] +fn test_deserialize_invalid_ommer_field_element() { + // Build a valid frontier, then replace one ommer with 0xFF bytes. + let mut f = CommitmentFrontier::new(); + for i in 0..4u64 { + f.append(test_leaf(i)).value.expect("append"); } + let mut data = f.serialize(); + let ommer_count = data[42] as usize; + assert!(ommer_count > 0, "need at least one ommer"); + // First ommer starts at byte 43, replace it with all 0xFF + for b in &mut data[43..43 + 32] { + *b = 0xFF; + } + let err = CommitmentFrontier::deserialize(&data); + assert!(err.is_err(), "should fail on invalid ommer field element"); + let msg = format!("{}", err.expect_err("should be an error")); + assert!( + msg.contains("invalid Pallas field element"), + "expected InvalidFieldElement error, got: {msg}" + ); +} + +#[test] +fn test_deserialize_from_parts_failure() { + // Construct technically valid field elements but with an inconsistent + // position/ommer combination that `Frontier::from_parts` rejects. + // Position 0 should have 0 ommers; providing 1 ommer triggers the + // from_parts validation error. + let valid_leaf = test_leaf(0); + let valid_ommer = test_leaf(1); + + let mut data = vec![0x01]; // has_frontier + data.extend_from_slice(&0u64.to_be_bytes()); // position = 0 + data.extend_from_slice(&valid_leaf); // leaf + data.push(1); // ommer_count = 1 (wrong for position 0) + data.extend_from_slice(&valid_ommer); // ommer + + let err = CommitmentFrontier::deserialize(&data); + assert!(err.is_err(), "should fail on inconsistent from_parts"); + let msg = format!("{}", err.expect_err("should be an error")); + assert!( + msg.contains("frontier reconstruction"), + "expected 'frontier reconstruction' error, got: {msg}" + ); +} + +#[test] +fn test_append_cost_sinsemilla_hash_calls() { + let mut f = CommitmentFrontier::new(); + + // First append (position 0): 32 hashes + 0 trailing_ones(empty) = 32 + let r0 = f.append(test_leaf(0)); + r0.value.expect("append 0"); + assert_eq!(r0.cost.sinsemilla_hash_calls, 32); + + // Second append (position 0 in frontier before append): trailing_ones(0) + // = 0 0 in binary is ...0, trailing_ones = 0, so 32 + 0 = 32 + let r1 = f.append(test_leaf(1)); + r1.value.expect("append 1"); + assert_eq!(r1.cost.sinsemilla_hash_calls, 32); + + // Third append (position 1): trailing_ones(1) = 1, so 32 + 1 = 33 + let r2 = f.append(test_leaf(2)); + r2.value.expect("append 2"); + assert_eq!(r2.cost.sinsemilla_hash_calls, 33); + + // Fourth append (position 2): trailing_ones(2=0b10) = 0, so 32 + let r3 = f.append(test_leaf(3)); + r3.value.expect("append 3"); + assert_eq!(r3.cost.sinsemilla_hash_calls, 32); + + // Fifth append (position 3): trailing_ones(3=0b11) = 2, so 34 + let r4 = f.append(test_leaf(4)); + r4.value.expect("append 4"); + assert_eq!(r4.cost.sinsemilla_hash_calls, 34); +} + +#[test] +fn test_deserialize_invalid_frontier_flag() { + // Test with a frontier flag value that is neither 0x00 nor 0x01 + let err = CommitmentFrontier::deserialize(&[0x42]); + assert!(err.is_err()); + let msg = format!("{}", err.expect_err("should be an error")); + assert!( + msg.contains("invalid frontier flag: 0x42"), + "expected 'invalid frontier flag' error, got: {msg}" + ); } diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs index f8ac5f7f9..0769efb75 100644 --- a/grovedb-commitment-tree/src/commitment_tree/mod.rs +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -285,14 +285,14 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { value: Ok(root), cost: frontier_cost, } => { - cost = cost + frontier_cost; + cost += frontier_cost; root } grovedb_costs::CostContext { value: Err(e), cost: frontier_cost, } => { - cost = cost + frontier_cost; + cost += frontier_cost; return Err(e).wrap_with_cost(cost); } }; diff --git a/grovedb-commitment-tree/src/error.rs b/grovedb-commitment-tree/src/error.rs index 3f1ebbbdd..41827de87 100644 --- a/grovedb-commitment-tree/src/error.rs +++ b/grovedb-commitment-tree/src/error.rs @@ -4,12 +4,22 @@ use thiserror::Error; /// Errors that can occur during commitment tree operations. #[derive(Debug, Error)] pub enum CommitmentTreeError { + /// The commitment tree has reached its maximum capacity (2^32 leaves). #[error("tree is full (max {max} leaves)", max = 1u64 << NOTE_COMMITMENT_TREE_DEPTH)] TreeFull, + /// Data read from storage is invalid or corrupt. #[error("invalid frontier data: {0}")] InvalidData(String), + /// A 32-byte value is not a valid Pallas field element. #[error("invalid Pallas field element")] InvalidFieldElement, + /// The ciphertext payload length does not match the expected size for the + /// configured `MemoSize`. #[error("invalid payload size: expected {expected}, got {actual}")] - InvalidPayloadSize { expected: usize, actual: usize }, + InvalidPayloadSize { + /// Expected payload byte length. + expected: usize, + /// Actual payload byte length received. + actual: usize, + }, } diff --git a/grovedb-commitment-tree/src/lib.rs b/grovedb-commitment-tree/src/lib.rs index 09cb3f083..80a080bd1 100644 --- a/grovedb-commitment-tree/src/lib.rs +++ b/grovedb-commitment-tree/src/lib.rs @@ -1,3 +1,4 @@ +#![warn(missing_docs)] //! Orchard-style commitment tree integration for GroveDB. //! //! This crate provides a lightweight frontier-based Sinsemilla Merkle tree From 29c51f398f246f7da04530287298b1d69700b0b9 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 26 Feb 2026 18:23:32 +0700 Subject: [PATCH 6/6] more work --- .../src/client/sqlite_store/tree_serialization.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/grovedb-commitment-tree/src/client/sqlite_store/tree_serialization.rs b/grovedb-commitment-tree/src/client/sqlite_store/tree_serialization.rs index 4300494ab..103c97a42 100644 --- a/grovedb-commitment-tree/src/client/sqlite_store/tree_serialization.rs +++ b/grovedb-commitment-tree/src/client/sqlite_store/tree_serialization.rs @@ -111,7 +111,11 @@ fn deserialize_tree_bounded( "invalid Pallas field element in leaf".to_string(), ) })?; - let flags = RetentionFlags::from_bits_truncate(flags_byte); + let flags = RetentionFlags::from_bits(flags_byte).ok_or_else(|| { + SqliteShardStoreError::Serialization(format!( + "invalid retention flags: 0x{flags_byte:02x}" + )) + })?; Ok(Tree::leaf((hash, flags))) } TAG_PARENT => {