diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index 3e3d069b2..f21c94eae 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -59,6 +59,10 @@ rand = { workspace = true } rand_distr = "0.6" assert_matches = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "time", "sync"] } +# For the `unsafe-dump-load` dump+ingest roundtrip test: writing the SST and +# ingesting it both need direct rocksdb types. Pinned to the same revision +# used by `grovedb-storage`'s optional dep. +rocksdb = { git = "https://github.com/QuantumExplorer/rust-rocksdb.git", rev = "52772eea7bcd214d1d07d80aa538b1d24e5015b7" } [[bench]] name = "insertion_benchmark" @@ -119,6 +123,17 @@ grovedbg = [ "zip-extensions", "tempfile", ] +# Snapshot-bootstrap primitives — escape hatches that bypass grovedb's normal +# validation: +# - `GroveDb::raw_storage` (escape hatch to StorageContext) +# - `RocksDbStorage::ingest_subtree_sst` (bulk-load SST into a CF) +# - `GroveDb::replace_subtree_root` (caller-provided child hash, no +# cross-check against actual subtree state) +# Enable ONLY for build-time tooling that needs to dump+restore a subtree +# out-of-band (e.g. devnet shielded-pool snapshot bake/apply). Misuse will +# produce an inconsistent Merk tree — there is no runtime check that the +# caller-provided hash matches the underlying state. +unsafe-dump-load = ["grovedb-storage/unsafe-dump-load"] [build-dependencies] hex-literal = "1.1.0" diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 6a77abf7b..485a9cb33 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -343,6 +343,54 @@ impl GroveDb { Ok(()) } + /// Reborrow the underlying [`grovedb_storage::rocksdb_storage::RocksDbStorage`] + /// for callers that need to use the public [`grovedb_storage::Storage`] + /// trait directly — notably to open a [`grovedb_storage::StorageContext`] + /// at a path for raw iteration or low-level reads. + /// + /// This is intended for snapshot/replication tooling that needs to walk + /// a subtree's raw RocksDB state without going through GroveDb's typed + /// element API. Normal callers should NOT use this — go through GroveDb's + /// typed operations (`insert`, `get`, `commitment_tree_*`) instead. + /// + /// Stability: this is an escape hatch. The exact `RocksDbStorage` shape + /// is subject to change as grovedb's internals evolve. + /// + /// Gated behind the `unsafe-dump-load` feature — production builds should + /// leave it off so this escape hatch isn't even compiled in. + #[cfg(feature = "unsafe-dump-load")] + pub fn raw_storage(&self) -> &grovedb_storage::rocksdb_storage::RocksDbStorage { + &self.db + } + + /// Bulk-ingest a single SST file (produced by `rocksdb::SstFileWriter`) + /// into the named column family of the underlying RocksDB. + /// + /// Delegates to + /// [`grovedb_storage::rocksdb_storage::RocksDbStorage::ingest_subtree_sst`]. + /// Intended for snapshot-based bootstrap of a single subtree's storage + /// state — see that method's docs for safety contract. + /// + /// CF name for ordinary data storage is the default CF + /// (`rocksdb::DEFAULT_COLUMN_FAMILY_NAME`). Aux/roots/meta CFs are also + /// valid targets if a snapshot tool happens to cover them. + /// + /// This call bypasses any open transaction. The caller is responsible for + /// transaction semantics at a higher layer (e.g. only call when the + /// destination subtree is known empty, and rely on InitChain + /// abort = wipe-and-restart for failure recovery). + /// + /// Gated behind the `unsafe-dump-load` feature. + #[cfg(feature = "unsafe-dump-load")] + pub fn ingest_subtree_sst( + &self, + cf_name: &str, + sst_path: &std::path::Path, + ) -> Result<(), Error> { + self.db.ingest_subtree_sst(cf_name, sst_path)?; + Ok(()) + } + /// Opens the transactional Merk at the given path. Returns CostResult. fn open_transactional_merk_at_path<'db, 'b, B>( &'db self, diff --git a/grovedb/src/operations/mod.rs b/grovedb/src/operations/mod.rs index 2f22aded6..d9e383c03 100644 --- a/grovedb/src/operations/mod.rs +++ b/grovedb/src/operations/mod.rs @@ -26,5 +26,11 @@ pub mod bulk_append_tree; #[cfg(feature = "minimal")] pub mod dense_tree; +/// Caller-driven subtree-root replacement. Bypasses grovedb's normal +/// "compute child hash from subtree state" invariant — see the module-level +/// docs in `replace_subtree_root.rs` for the safety contract. +#[cfg(all(feature = "minimal", feature = "unsafe-dump-load"))] +pub mod replace_subtree_root; + #[cfg(feature = "minimal")] pub use get::{QueryItemOrSumReturnType, MAX_REFERENCE_HOPS}; diff --git a/grovedb/src/operations/replace_subtree_root.rs b/grovedb/src/operations/replace_subtree_root.rs new file mode 100644 index 000000000..3395f71b3 --- /dev/null +++ b/grovedb/src/operations/replace_subtree_root.rs @@ -0,0 +1,120 @@ +//! Caller-driven subtree-root replacement. +//! +//! Replaces a subtree leaf's child hash + element data inside its parent +//! Merk with caller-provided values, **without** re-reading the subtree to +//! compute the hash. This is the tail of every `*_insert` operation +//! (commitment_tree, mmr_tree, bulk_append_tree, dense_tree) factored out as +//! a single generic helper, intended for **snapshot-based bootstrap** only. +//! +//! # Safety contract +//! +//! The caller is responsible for ensuring `new_combined_root` actually +//! matches the subtree's underlying storage state. Intended pattern: +//! +//! 1. Ingest the subtree's raw storage out-of-band (typically via +//! [`grovedb_storage::rocksdb_storage::RocksDbStorage::ingest_subtree_sst`]). +//! 2. Open a `StorageContext` at the subtree path and reconstruct the +//! appropriate typed tree (CommitmentTree / Mmr / BulkAppend / Dense). +//! 3. Compute the post-ingest combined root via that tree's own root fn. +//! 4. Call [`GroveDb::replace_subtree_root`] with the verified hash and a +//! correctly-constructed [`Element`] (e.g. `Element::new_commitment_tree`). +//! +//! Mismatches between `new_combined_root` and what re-reading the subtree +//! would compute produce an inconsistent Merk tree (the parent's recorded +//! child hash diverges from actual subtree state). This method does NOT +//! detect such mismatches at runtime; misuse is silent corruption. +//! +//! Gated behind the `unsafe-dump-load` Cargo feature. + +use std::collections::HashMap; + +use grovedb_costs::{ + cost_return_on_error, cost_return_on_error_into, CostResult, CostsExt, OperationCost, +}; +use grovedb_merk::element::{ + insert::ElementInsertToStorageExtensions, tree_type::ElementTreeTypeExtensions, +}; +use grovedb_path::SubtreePath; +use grovedb_storage::{Storage, StorageBatch}; +use grovedb_version::version::GroveVersion; + +use crate::{util::TxRef, Element, Error, GroveDb, TransactionArg}; + +impl GroveDb { + /// Replace the child hash + element data of a subtree leaf in its parent + /// Merk with caller-provided values. See the + /// [module-level docs](self) for the safety contract. + pub fn replace_subtree_root<'b, B, P>( + &self, + path: P, + key: &[u8], + new_element: Element, + new_combined_root: [u8; 32], + transaction: TransactionArg, + grove_version: &GroveVersion, + ) -> CostResult<(), Error> + where + B: AsRef<[u8]> + 'b, + P: Into>, + { + let mut cost = OperationCost::default(); + + // Cheap sanity check: must be a tree-shaped Element (Item / Reference + // have no child hash slot). Caller still owns hash-vs-state + // correctness. + if new_element.tree_type().is_none() { + return Err(Error::InvalidInput( + "replace_subtree_root: element is not a tree variant", + )) + .wrap_with_cost(cost); + } + + let path: SubtreePath = path.into(); + let tx = TxRef::new(&self.db, transaction); + let batch = StorageBatch::new(); + + let mut parent_merk = cost_return_on_error!( + &mut cost, + self.open_transactional_merk_at_path( + path.clone(), + tx.as_ref(), + Some(&batch), + grove_version, + ) + ); + + cost_return_on_error_into!( + &mut cost, + new_element.insert_subtree( + &mut parent_merk, + key, + new_combined_root, + None, + grove_version, + ) + ); + + let mut merk_cache = HashMap::new(); + merk_cache.insert(path.clone(), parent_merk); + + cost_return_on_error!( + &mut cost, + self.propagate_changes_with_transaction( + merk_cache, + path, + tx.as_ref(), + &batch, + grove_version, + ) + ); + + cost_return_on_error!( + &mut cost, + self.db + .commit_multi_context_batch(batch, Some(tx.as_ref())) + .map_err(Into::into) + ); + + tx.commit_local().wrap_with_cost(cost) + } +} diff --git a/grovedb/src/tests/commitment_tree_tests.rs b/grovedb/src/tests/commitment_tree_tests.rs index 06b86bf5c..ab984898c 100644 --- a/grovedb/src/tests/commitment_tree_tests.rs +++ b/grovedb/src/tests/commitment_tree_tests.rs @@ -20,6 +20,17 @@ use crate::{ Element, Error, GroveDb, PathQuery, SizedQuery, }; +#[cfg(feature = "unsafe-dump-load")] +use grovedb_commitment_tree::CommitmentTree; +#[cfg(feature = "unsafe-dump-load")] +use grovedb_path::SubtreePath; +#[cfg(feature = "unsafe-dump-load")] +use grovedb_storage::{rocksdb_storage::RocksDbStorage, RawIterator, Storage, StorageContext}; +#[cfg(feature = "unsafe-dump-load")] +use rocksdb::{Options, SstFileWriter}; +#[cfg(feature = "unsafe-dump-load")] +use tempfile::TempDir; + /// Default chunk power for tests (2^10 = 1024, large enough that compaction /// doesn't happen in most tests with only a few items). const TEST_CHUNK_POWER: u8 = 10; @@ -2496,3 +2507,177 @@ fn test_commitment_compaction_transaction_rollback() { "anchor should revert to empty tree after compaction + rollback" ); } + +// =========================================================================== +// unsafe-dump-load: dump-then-restore subtree round-trip +// =========================================================================== +// +// Exercises the three escape-hatch APIs gated behind `unsafe-dump-load` +// (`GroveDb::raw_storage`, `RocksDbStorage::ingest_subtree_sst`, +// `GroveDb::replace_subtree_root`) in a single end-to-end round-trip on a +// real CommitmentTree subtree. The post-restore GroveDb root_hash must equal +// the source's, byte-for-byte, demonstrating the snapshot-bootstrap contract +// described in `replace_subtree_root.rs`'s module docs. + +#[cfg(feature = "unsafe-dump-load")] +#[test] +fn unsafe_dump_load_subtree_roundtrip_preserves_root_hash() { + let grove_version = GroveVersion::latest(); + const N: u8 = 5; + + // --- A: build a commitment-tree subtree, insert N items --- + let db_a = make_empty_grovedb(); + db_a.insert( + EMPTY_PATH, + b"pool", + Element::empty_commitment_tree(TEST_CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert empty ct on A"); + + for i in 0..N { + db_a.commitment_tree_insert( + EMPTY_PATH, + b"pool", + test_cmx(i + 1), + test_rho(i + 1), + test_ciphertext(i + 1), + None, + grove_version, + ) + .unwrap() + .expect("insert leaf on A"); + } + + let root_a = db_a + .root_hash(None, grove_version) + .unwrap() + .expect("source root_hash"); + + // --- Compute combined_root by re-opening CommitmentTree on A's storage --- + let pool_segments: [&[u8]; 1] = [b"pool"]; + let subtree_path = SubtreePath::from(pool_segments.as_slice()); + let prefix: [u8; 32] = RocksDbStorage::build_prefix(subtree_path.clone()).value; + + let tx_a = db_a.start_transaction(); + let ctx_open = db_a + .raw_storage() + .get_transactional_storage_context(subtree_path.clone(), None, &tx_a) + .unwrap(); + let ct_a = CommitmentTree::<_, DashMemo>::open(u64::from(N), TEST_CHUNK_POWER, ctx_open) + .value + .expect("CommitmentTree::open on A"); + let combined_root = ct_a + .compute_current_state_root() + .expect("compute_current_state_root on A"); + drop(ct_a); + + // --- Dump A's subtree to an SST (re-prepending the prefix that raw_iter strips) --- + let pool_segments_iter: [&[u8]; 1] = [b"pool"]; + let iter_path = SubtreePath::from(pool_segments_iter.as_slice()); + let iter_ctx = db_a + .raw_storage() + .get_transactional_storage_context(iter_path, None, &tx_a) + .unwrap(); + let sst_tmp = TempDir::new().expect("sst tempdir"); + let sst_path = sst_tmp.path().join("dump.sst"); + + let opts = Options::default(); + let mut sst = SstFileWriter::create(&opts); + sst.open(&sst_path).expect("SstFileWriter::open"); + + let mut iter = iter_ctx.raw_iter(); + iter.seek_to_first().unwrap(); + let mut key_count: u64 = 0; + while iter.valid().unwrap() { + let user_key = iter + .key() + .unwrap() + .expect("iter.key() during dump") + .to_vec(); + let value = iter + .value() + .unwrap() + .expect("iter.value() during dump") + .to_vec(); + let mut full_key = Vec::with_capacity(32 + user_key.len()); + full_key.extend_from_slice(&prefix); + full_key.extend_from_slice(&user_key); + sst.put(&full_key, &value).expect("SstFileWriter::put"); + key_count += 1; + iter.next().unwrap(); + } + sst.finish().expect("SstFileWriter::finish"); + assert!(key_count > 0, "dumped SST must contain at least one key"); + drop(iter); + drop(iter_ctx); + drop(tx_a); + drop(db_a); + + // --- B: fresh GroveDb with same empty commitment_tree skeleton --- + let db_b = make_empty_grovedb(); + db_b.insert( + EMPTY_PATH, + b"pool", + Element::empty_commitment_tree(TEST_CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert empty ct on B"); + + db_b.ingest_subtree_sst("default", &sst_path) + .expect("ingest_subtree_sst"); + + let restored_element = Element::new_commitment_tree(u64::from(N), TEST_CHUNK_POWER, None); + db_b.replace_subtree_root( + EMPTY_PATH, + b"pool", + restored_element, + combined_root, + None, + grove_version, + ) + .unwrap() + .expect("replace_subtree_root"); + + let root_b = db_b + .root_hash(None, grove_version) + .unwrap() + .expect("restored root_hash"); + + assert_eq!( + root_a, root_b, + "post-restore GroveDB root_hash must match source byte-for-byte" + ); +} + +#[cfg(feature = "unsafe-dump-load")] +#[test] +fn replace_subtree_root_rejects_non_tree_element() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"pool", + Element::empty_commitment_tree(TEST_CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert ct"); + + let item = Element::new_item(b"not a tree".to_vec()); + let result = db + .replace_subtree_root(EMPTY_PATH, b"pool", item, [0u8; 32], None, grove_version) + .unwrap(); + assert!( + matches!(result, Err(Error::InvalidInput(_))), + "expected InvalidInput for non-tree element, got {result:?}" + ); +} diff --git a/storage/Cargo.toml b/storage/Cargo.toml index f76532bb2..89e3906c5 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -26,3 +26,7 @@ thiserror = { workspace = true } [features] rocksdb_storage = ["rocksdb", "num_cpus", "lazy_static", "tempfile", "blake3", "integer-encoding"] +# See `grovedb`'s `unsafe-dump-load` feature for context. On this crate it +# gates `RocksDbStorage::ingest_subtree_sst` — the RocksDB IngestExternalFile +# wrapper used to bulk-load a precomputed subtree SST. +unsafe-dump-load = ["rocksdb_storage"] diff --git a/storage/src/rocksdb_storage/storage.rs b/storage/src/rocksdb_storage/storage.rs index 9ff20c4c4..96edafbb1 100644 --- a/storage/src/rocksdb_storage/storage.rs +++ b/storage/src/rocksdb_storage/storage.rs @@ -39,6 +39,8 @@ use grovedb_costs::{ use grovedb_path::SubtreePath; use integer_encoding::VarInt; use lazy_static::lazy_static; +#[cfg(feature = "unsafe-dump-load")] +use rocksdb::IngestExternalFileOptions; use rocksdb::{ checkpoint::Checkpoint, ColumnFamily, ColumnFamilyDescriptor, OptimisticTransactionDB, Transaction, WriteBatchWithTransaction, DEFAULT_COLUMN_FAMILY_NAME, @@ -494,6 +496,50 @@ impl RocksDbStorage { } } + /// Bulk-ingest a single SST file (produced by `rocksdb::SstFileWriter`) + /// into the named column family. + /// + /// Used by snapshot-based bootstrap (e.g. the shielded-pool genesis + /// snapshot) to load a precomputed subtree's keys without paying the + /// per-write WAL + fsync cost. The SST must be sorted and its key range + /// must NOT overlap with any keys already in the CF — otherwise ingest + /// fails. For genesis-time usage this is satisfied by definition (the + /// target subtree is empty when this is called). + /// + /// Security notes (set by this method, not caller-configurable): + /// - `allow_global_seqno=false`: rejects ingests that would inject a + /// global sequence number, preventing a malicious snapshot from + /// reordering its writes relative to subsequent transactional writes. + /// - `snapshot_consistency=false`: snapshot-based bootstrap runs before + /// any reader could hold a RocksDB snapshot of the empty state. + /// + /// The ingest happens at the DB level and bypasses any open transaction. + /// Callers must arrange for txn semantics at a higher layer. + /// + /// Gated behind the `unsafe-dump-load` feature — production builds (which + /// have no need to bulk-load precomputed subtree state) should leave it + /// off so this API isn't even compiled in. + #[cfg(feature = "unsafe-dump-load")] + pub fn ingest_subtree_sst(&self, cf_name: &str, sst_path: &Path) -> Result<(), Error> { + let cf_handle = self + .db + .cf_handle(cf_name) + .ok_or(Error::StorageError(format!( + "ingest_subtree_sst: missing CF {cf_name}" + )))?; + let mut opts = IngestExternalFileOptions::default(); + opts.set_allow_global_seqno(false); + opts.set_snapshot_consistency(false); + self.db + .ingest_external_file_cf_opts(&cf_handle, &opts, vec![sst_path]) + .map_err(|e| { + Error::StorageError(format!( + "ingest_subtree_sst({cf_name}, {}) failed: {e}", + sst_path.display() + )) + }) + } + /// Clears all data from the database using range deletion on each /// column family. Uses a single range tombstone per CF instead of /// iterating and deleting every key individually.