feat: use transactional storage context for all specialized trees

docs/book/src/commitment-tree.md

@@ -330,8 +330,10 @@ db.commitment_tree_count(path, key, tx, version)
 
 The typed `commitment_tree_insert` accepts a `TransmittedNoteCiphertext<M>` and
 serializes it internally. The raw `commitment_tree_insert_raw` (pub(crate))
-accepts `Vec<u8>` and is used by batch preprocessing where payloads are already
-serialized.
+accepts `Vec<u8>` and is used by the direct (non-batch) API and
+`apply_operations_without_batching`. Batch preprocessing performs its own
+inline processing using a shared `StorageBatch` and does not call
+`commitment_tree_insert_raw`.
 
 ### commitment_tree_insert
 

grovedb-bulk-append-tree/src/proof/mod.rs

@@ -14,7 +14,7 @@ use bincode::{Decode, Encode};
 use grovedb_dense_fixed_sized_merkle_tree::DenseTreeProof;
 use grovedb_merkle_mountain_range::MmrTreeProof;
 #[cfg(feature = "storage")]
-use grovedb_merkle_mountain_range::{MMRStoreReadOps, MmrKeySize, MmrNode, MmrStore};
+use grovedb_merkle_mountain_range::{MmrKeySize, MmrNode, MmrStore, MMR};
 use grovedb_query::{Query, QueryItem};
 #[cfg(feature = "storage")]
 use grovedb_storage::StorageContext;
@@ -239,8 +239,9 @@ impl BulkAppendTreeProof {
             let chunk_indices: Vec<u64> = chunk_indices_set.into_iter().collect();
 
             let mmr_store = MmrStore::with_key_size(&tree.dense_tree.storage, MmrKeySize::U32);
+            let mmr = MMR::new_with_overlay(mmr_size, &mmr_store, tree.mmr_overlay.clone());
             let get_node = |pos: u64| -> grovedb_merkle_mountain_range::Result<Option<MmrNode>> {
-                (&mmr_store).element_at_position(pos).unwrap() // unwrap CostResult
+                mmr.batch.element_at_position(pos).unwrap() // unwrap CostResult
             };
 
             MmrTreeProof::generate(mmr_size, &chunk_indices, get_node).map_err(|e| {

grovedb-bulk-append-tree/src/tree/append.rs

@@ -19,6 +19,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree<S> {
         Ok(Self {
             total_count: 0,
             dense_tree,
+            mmr_overlay: Vec::new(),
         })
     }
 
@@ -42,6 +43,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree<S> {
         Ok(Self {
             total_count,
             dense_tree,
+            mmr_overlay: Vec::new(),
         })
     }
 
@@ -136,24 +138,36 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree<S> {
         let leaf_count = mmr_size_to_leaf_count(mmr_size);
         hash_count += hash_count_for_push(leaf_count);
 
-        // Create MmrStore on the fly from the dense tree's storage
+        // Create MmrStore on the fly from the dense tree's storage.
+        // Use the overlay from previous compactions so cross-compaction
+        // reads work without a storage round-trip. After push+get_root,
+        // take the overlay back (don't commit — that happens at session end).
         let mmr_root = {
             let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32);
-            let mut mmr = MMR::new(mmr_size, &mmr_store);
-            mmr.push(leaf)
-                .unwrap()
-                .map_err(|e| BulkAppendError::MmrError(format!("MMR push failed: {}", e)))?;
-
-            // Get root BEFORE commit — data is still in the MMRBatch overlay
-            let root_node = mmr
-                .get_root()
-                .unwrap()
-                .map_err(|e| BulkAppendError::MmrError(format!("MMR get_root failed: {}", e)))?;
-            let root = root_node.hash();
+            let mut mmr =
+                MMR::new_with_overlay(mmr_size, &mmr_store, std::mem::take(&mut self.mmr_overlay));
+
+            let push_result = mmr.push(leaf).unwrap();
+            if let Err(e) = push_result {
+                // Restore overlay before returning error
+                self.mmr_overlay = mmr.batch.take_overlay();
+                return Err(BulkAppendError::MmrError(format!("MMR push failed: {}", e)));
+            }
 
-            mmr.commit()
-                .unwrap()
-                .map_err(|e| BulkAppendError::MmrError(format!("MMR commit failed: {}", e)))?;
+            let root_result = mmr.get_root().unwrap();
+            let root = match root_result {
+                Ok(node) => node.hash(),
+                Err(e) => {
+                    self.mmr_overlay = mmr.batch.take_overlay();
+                    return Err(BulkAppendError::MmrError(format!(
+                        "MMR get_root failed: {}",
+                        e
+                    )));
+                }
+            };
+
+            // Take overlay back instead of committing
+            self.mmr_overlay = mmr.batch.take_overlay();
 
             root
         };
@@ -171,11 +185,42 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree<S> {
             return Ok([0u8; 32]);
         }
         let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32);
-        let mmr = MMR::new(mmr_size, &mmr_store);
+        let mmr = MMR::new_with_overlay(mmr_size, &mmr_store, self.mmr_overlay.clone());
         let root_node = mmr
             .get_root()
             .unwrap()
             .map_err(|e| BulkAppendError::MmrError(format!("MMR get_root failed: {}", e)))?;
         Ok(root_node.hash())
     }
+
+    /// Flush the MMR overlay to storage.
+    ///
+    /// Call this at the end of a session to persist all MMR nodes that were
+    /// buffered during compaction cycles. This is a no-op if no compactions
+    /// occurred.
+    ///
+    /// Cost tracking is intentionally omitted at this boundary:
+    /// BulkAppendTree returns plain `Result`, not `CostResult`. Storage
+    /// I/O costs are captured by the caller's `commit_multi_context_batch`.
+    pub fn commit_mmr(&mut self) -> Result<(), BulkAppendError> {
+        if self.mmr_overlay.is_empty() {
+            return Ok(());
+        }
+        let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32);
+        let mut mmr = MMR::new_with_overlay(
+            self.mmr_size(),
+            &mmr_store,
+            std::mem::take(&mut self.mmr_overlay),
+        );
+        if let Err(e) = mmr.commit().unwrap() {
+            // Restore overlay before returning error so retries/get_mmr_root
+            // still see the staged nodes.
+            self.mmr_overlay = mmr.batch.take_overlay();
+            return Err(BulkAppendError::MmrError(format!(
+                "MMR commit failed: {}",
+                e
+            )));
+        }
+        Ok(())
+    }
 }

grovedb-bulk-append-tree/src/tree/fetch.rs

@@ -1,7 +1,7 @@
 //! Read operations for BulkAppendTree.
 
 use grovedb_dense_fixed_sized_merkle_tree::DenseTreeProof;
-use grovedb_merkle_mountain_range::{leaf_to_pos, MMRStoreReadOps, MmrKeySize, MmrStore, MMR};
+use grovedb_merkle_mountain_range::{leaf_to_pos, MmrKeySize, MmrStore, MMR};
 use grovedb_query::Query;
 use grovedb_storage::StorageContext;
 
@@ -69,13 +69,18 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree<S> {
     ///
     /// This reads from the **chunk MMR**, which stores immutable epoch blobs.
     /// Returns `None` if the chunk hasn't been completed yet.
+    ///
+    /// Uses the MMR overlay to find nodes that were pushed during this session
+    /// but not yet committed to storage.
     pub fn get_chunk_value(&self, chunk_index: u64) -> Result<Option<Vec<u8>>, BulkAppendError> {
         if chunk_index >= self.chunk_count() {
             return Ok(None);
         }
         let mmr_pos = leaf_to_pos(chunk_index);
         let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32);
-        let node = (&mmr_store)
+        let mmr = MMR::new_with_overlay(self.mmr_size(), &mmr_store, self.mmr_overlay.clone());
+        let node = mmr
+            .batch
             .element_at_position(mmr_pos)
             .unwrap()
             .map_err(|e| {
@@ -130,7 +135,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree<S> {
             (Vec::new(), [0u8; 32])
         } else {
             let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32);
-            let mmr = MMR::new(mmr_size, &mmr_store);
+            let mmr = MMR::new_with_overlay(mmr_size, &mmr_store, self.mmr_overlay.clone());
 
             let positions: Vec<u64> = chunk_indices.iter().map(|&idx| leaf_to_pos(idx)).collect();
             let proof = mmr.gen_proof(positions).unwrap().map_err(|e| {

grovedb-bulk-append-tree/src/tree/mod.rs

@@ -21,6 +21,7 @@ pub use fetch::{BufferQueryResult, ChunkQueryResult};
 mod tests;
 
 use grovedb_dense_fixed_sized_merkle_tree::DenseFixedSizedMerkleTree;
+use grovedb_merkle_mountain_range::MmrNode;
 
 #[cfg(feature = "storage")]
 use crate::BulkAppendError;
@@ -69,6 +70,11 @@ pub struct BulkAppendTree<S> {
     /// the MMR size and dense tree state.
     pub total_count: u64,
     pub dense_tree: DenseFixedSizedMerkleTree<S>,
+    /// MMR node overlay: holds nodes pushed during this session that have
+    /// not yet been committed to storage. Persists across MMR instance
+    /// lifetimes (compaction cycles) so that reads can find recently-pushed
+    /// nodes without a storage round-trip.
+    pub(crate) mmr_overlay: Vec<(u64, Vec<MmrNode>)>,
 }
 
 impl<S> BulkAppendTree<S> {

grovedb-commitment-tree/src/commitment_tree/mod.rs

@@ -366,6 +366,16 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree<S, M> {
 
     // ── BulkAppendTree delegates ──────────────────────────────────────
 
+    /// Flush the MMR overlay to storage.
+    ///
+    /// Delegates to [`BulkAppendTree::commit_mmr`]. Call this at the end of a
+    /// session to persist MMR nodes buffered during compaction cycles.
+    pub fn commit_mmr(&mut self) -> Result<(), CommitmentTreeError> {
+        self.bulk_tree
+            .commit_mmr()
+            .map_err(|e| CommitmentTreeError::InvalidData(format!("MMR commit: {}", e)))
+    }
+
     /// Get the total count of items appended (from the BulkAppendTree).
     pub fn total_count(&self) -> u64 {
         self.bulk_tree.total_count

grovedb-dense-fixed-sized-merkle-tree/src/tests.rs

@@ -681,8 +681,14 @@ fn test_get_store_inconsistency_errors() {
 
 #[test]
 fn test_proof_generate_store_failure() {
-    let mut tree = DenseFixedSizedMerkleTree::new(3, MemStorageContext::new()).expect("height 3");
-    tree.insert(b"val").unwrap().expect("insert should succeed");
+    // Use from_state (not insert) so the write-through cache is empty.
+    // Then corrupt the store to simulate missing data.
+    let store = MemStorageContext::new();
+    store
+        .data
+        .borrow_mut()
+        .insert(position_key(0).to_vec(), b"val".to_vec());
+    let tree = DenseFixedSizedMerkleTree::from_state(3, 1, store).expect("from_state");
 
     // Corrupt the store by removing the value
     tree.storage

grovedb-dense-fixed-sized-merkle-tree/src/tree.rs

@@ -33,6 +33,11 @@ pub fn position_key(pos: u16) -> [u8; 2] {
 ///
 /// Storage is embedded directly on the struct (like Merk).
 ///
+/// A write-through cache (`cache`) holds values written during this session.
+/// Reads check the cache first, falling back to storage. This enables use
+/// with transactional storage contexts where writes are deferred to a batch
+/// and not yet visible through reads.
+///
 /// Note: root hash computation is O(n) per insert where n = count, since no
 /// intermediate hashes are cached. Suitable for small trees (epoch sizes
 /// typically 16-256).
@@ -41,6 +46,12 @@ pub struct DenseFixedSizedMerkleTree<S> {
     count: u16,
     /// The underlying storage context.
     pub storage: S,
+    /// Write-through cache: holds values written in this session.
+    /// Indexed by position. `None` means the value has not been written
+    /// in this session (fall back to storage).
+    /// Only compiled when storage-dependent operations are available.
+    #[cfg(feature = "storage")]
+    cache: Vec<Option<Vec<u8>>>,
 }
 
 // ── Pure accessors (no storage bounds needed) ─────────────────────────
@@ -77,14 +88,20 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree<S> {
     /// Height must be between 1 and 16 inclusive.
     pub fn new(height: u8, storage: S) -> Result<Self, DenseMerkleError> {
         validate_height(height)?;
+        let capacity = Self::capacity_for_height(height);
         Ok(Self {
             height,
             count: 0,
             storage,
+            cache: vec![None; capacity as usize],
         })
     }
 
     /// Reconstitute a tree from stored state.
+    ///
+    /// The cache starts empty — pre-existing values are loaded from storage
+    /// on demand. Only values written via [`insert`] or [`try_insert`] in
+    /// this session are cached.
     pub fn from_state(height: u8, count: u16, storage: S) -> Result<Self, DenseMerkleError> {
         validate_height(height)?;
         let capacity = Self::capacity_for_height(height);
@@ -98,6 +115,7 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree<S> {
             height,
             count,
             storage,
+            cache: vec![None; capacity as usize],
         })
     }
 
@@ -127,10 +145,13 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree<S> {
             // succeeds, which needs a storage fault (get fails on a key that was
             // just written). Not reachable with any StorageContext implementation.
             Err(e) => {
-                // Roll back count so the tree state remains consistent.
-                // Note: the value remains in the store; the caller is
-                // responsible for store-level cleanup if needed.
+                // Roll back count and cache so the tree state remains
+                // consistent. The value remains in the store; the caller
+                // is responsible for store-level cleanup if needed.
                 self.count -= 1;
+                if let Some(slot) = self.cache.get_mut(position as usize) {
+                    *slot = None;
+                }
                 Err(e).wrap_with_cost(cost)
             }
         }
@@ -158,6 +179,9 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree<S> {
             Ok(root_hash) => Ok(Some((root_hash, position))).wrap_with_cost(cost),
             Err(e) => {
                 self.count -= 1;
+                if let Some(slot) = self.cache.get_mut(position as usize) {
+                    *slot = None;
+                }
                 Err(e).wrap_with_cost(cost)
             }
         }
@@ -204,16 +228,31 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree<S> {
 
     /// Reset the tree to empty state.
     ///
-    /// Sets count to 0. Old values remain in the underlying storage (they
-    /// will be overwritten on the next cycle).
+    /// Sets count to 0 and clears the write-through cache. Old values
+    /// remain in the underlying storage (they will be overwritten on the
+    /// next cycle).
     pub fn reset(&mut self) {
         self.count = 0;
+        self.cache.fill(None);
     }
 
     // ── Internal storage helpers ──────────────────────────────────────
 
-    /// Read a value by position from storage.
+    /// Read a value by position, checking the write-through cache first.
+    ///
+    /// Cache hits return deterministic costs (seek_count=1,
+    /// storage_loaded_bytes=len) matching the MMRBatch pattern, so fee
+    /// estimates are consistent regardless of cache state.
     pub(crate) fn get_value(&self, position: u16) -> CostResult<Option<Vec<u8>>, DenseMerkleError> {
+        // Check write-through cache first
+        if let Some(Some(cached)) = self.cache.get(position as usize) {
+            return Ok(Some(cached.clone())).wrap_with_cost(OperationCost {
+                seek_count: 1,
+                storage_loaded_bytes: cached.len() as u64,
+                ..Default::default()
+            });
+        }
+        // Fall back to storage
         let mut cost = OperationCost::default();
         let key = position_key(position);
         let result = self.storage.get(key).unwrap_add_cost(&mut cost);
@@ -227,16 +266,32 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree<S> {
         }
     }
 
-    /// Write a value by position to storage.
-    fn put_value(&self, position: u16, value: &[u8]) -> CostResult<(), DenseMerkleError> {
+    /// Write a value by position to storage and cache.
+    ///
+    /// On success, the value is stored in the write-through cache so that
+    /// subsequent reads (e.g., during root hash computation) can be served
+    /// from memory even when the storage context defers writes.
+    fn put_value(&mut self, position: u16, value: &[u8]) -> CostResult<(), DenseMerkleError> {
+        debug_assert!(
+            (position as usize) < self.cache.len(),
+            "put_value called with position {} >= cache capacity {}",
+            position,
+            self.cache.len()
+        );
         let mut cost = OperationCost::default();
         let key = position_key(position);
         let result = self
             .storage
             .put(key, value, None, None)
             .unwrap_add_cost(&mut cost);
         match result {
-            Ok(()) => Ok(()).wrap_with_cost(cost),
+            Ok(()) => {
+                // Cache on successful write
+                if let Some(slot) = self.cache.get_mut(position as usize) {
+                    *slot = Some(value.to_vec());
+                }
+                Ok(()).wrap_with_cost(cost)
+            }
             Err(e) => Err(DenseMerkleError::StoreError(format!(
                 "put at pos {}: {}",
                 position, e