feat: Simple checkpoint

grovedb/src/lib.rs

@@ -10,11 +10,11 @@ use std::{
 
 use merk::{self, Merk};
 use rs_merkle::{algorithms::Sha256, MerkleTree};
-pub use subtree::Element;
 use storage::{
     rocksdb_storage::{PrefixedRocksDbStorage, PrefixedRocksDbStorageError},
     Storage,
 };
+pub use subtree::Element;
 
 /// Limit of possible indirections
 const MAX_REFERENCE_HOPS: usize = 10;
@@ -26,24 +26,18 @@ const ROOT_LEAFS_SERIALIZED_KEY: &[u8] = b"rootLeafsSerialized";
 
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[error("rocksdb error")]
-    RocksDBError(#[from] PrefixedRocksDbStorageError),
-    #[error("unable to open Merk db")]
-    MerkError(merk::Error),
-    #[error("invalid path: {0}")]
-    InvalidPath(&'static str),
-    #[error("unable to decode")]
-    BincodeError(#[from] bincode::Error),
+    // Input data errors
     #[error("cyclic reference path")]
     CyclicReference,
     #[error("reference hops limit exceeded")]
     ReferenceLimit,
-}
-
-impl From<merk::Error> for Error {
-    fn from(e: merk::Error) -> Self {
-        Error::MerkError(e)
-    }
+    #[error("invalid path: {0}")]
+    InvalidPath(&'static str),
+    // Irrecoverable errors
+    #[error("storage error: {0}")]
+    StorageError(#[from] PrefixedRocksDbStorageError),
+    #[error("data corruption error: {0}")]
+    CorruptedData(String),
 }
 
 pub struct GroveDb {
@@ -69,10 +63,14 @@ impl GroveDb {
         let mut subtrees = HashMap::new();
         // TODO: owned `get` is not required for deserialization
         if let Some(prefixes_serialized) = meta_storage.get_meta(SUBTRESS_SERIALIZED_KEY)? {
-            let subtrees_prefixes: Vec<Vec<u8>> = bincode::deserialize(&prefixes_serialized)?;
+            let subtrees_prefixes: Vec<Vec<u8>> = bincode::deserialize(&prefixes_serialized)
+                .map_err(|_| {
+                    Error::CorruptedData(String::from("unable to deserialize prefixes"))
+                })?;
             for prefix in subtrees_prefixes {
                 let subtree_merk =
-                    Merk::open(PrefixedRocksDbStorage::new(db.clone(), prefix.to_vec())?)?;
+                    Merk::open(PrefixedRocksDbStorage::new(db.clone(), prefix.to_vec())?)
+                        .map_err(|e| Error::CorruptedData(e.to_string()))?;
                 subtrees.insert(prefix.to_vec(), subtree_merk);
             }
         }
@@ -81,7 +79,9 @@ impl GroveDb {
         let root_leaf_keys: HashMap<Vec<u8>, usize> = if let Some(root_leaf_keys_serialized) =
             meta_storage.get_meta(ROOT_LEAFS_SERIALIZED_KEY)?
         {
-            bincode::deserialize(&root_leaf_keys_serialized)?
+            bincode::deserialize(&root_leaf_keys_serialized).map_err(|_| {
+                Error::CorruptedData(String::from("unable to deserialize root leafs"))
+            })?
         } else {
             HashMap::new()
         };
@@ -95,13 +95,25 @@ impl GroveDb {
         })
     }
 
+    pub fn checkpoint<P: AsRef<Path>>(&self, path: P) -> Result<GroveDb, Error> {
+        storage::rocksdb_storage::Checkpoint::new(&self.db)
+            .and_then(|x| x.create_checkpoint(&path))
+            .map_err(PrefixedRocksDbStorageError::RocksDbError)?;
+        GroveDb::open(path)
+    }
+
     fn store_subtrees_keys_data(&self) -> Result<(), Error> {
         let prefixes: Vec<Vec<u8>> = self.subtrees.keys().map(|x| x.clone()).collect();
-        self.meta_storage
-            .put_meta(SUBTRESS_SERIALIZED_KEY, &bincode::serialize(&prefixes)?)?;
+        self.meta_storage.put_meta(
+            SUBTRESS_SERIALIZED_KEY,
+            &bincode::serialize(&prefixes)
+                .map_err(|_| Error::CorruptedData(String::from("unable to serialize prefixes")))?,
+        )?;
         self.meta_storage.put_meta(
             ROOT_LEAFS_SERIALIZED_KEY,
-            &bincode::serialize(&self.root_leaf_keys)?,
+            &bincode::serialize(&self.root_leaf_keys).map_err(|_| {
+                Error::CorruptedData(String::from("unable to serialize root leafs"))
+            })?,
         )?;
         Ok(())
     }
@@ -140,7 +152,8 @@ impl GroveDb {
                             Merk::open(PrefixedRocksDbStorage::new(
                                 self.db.clone(),
                                 compressed_path_subtree,
-                            )?)?,
+                            )?)
+                            .map_err(|e| Error::CorruptedData(e.to_string()))?,
                         ))
                     };
                 if path.is_empty() {

grovedb/src/subtree.rs

@@ -30,10 +30,12 @@ impl Element {
     /// Merk should be loaded by this moment
     pub fn get(merk: &Merk<PrefixedRocksDbStorage>, key: &[u8]) -> Result<Element, Error> {
         let element = bincode::deserialize(
-            merk.get(&key)?
+            merk.get(&key)
+                .map_err(|e| Error::CorruptedData(e.to_string()))?
                 .ok_or(Error::InvalidPath("key not found in Merk"))?
                 .as_slice(),
-        )?;
+        )
+        .map_err(|_| Error::CorruptedData(String::from("unable to deserialize element")))?;
         Ok(element)
     }
 
@@ -44,8 +46,15 @@ impl Element {
         merk: &mut Merk<PrefixedRocksDbStorage>,
         key: Vec<u8>,
     ) -> Result<(), Error> {
-        let batch = [(key, Op::Put(bincode::serialize(self)?))];
-        merk.apply(&batch, &[]).map_err(|e| e.into())
+        let batch =
+            [(
+                key,
+                Op::Put(bincode::serialize(self).map_err(|_| {
+                    Error::CorruptedData(String::from("unable to serialize element"))
+                })?),
+            )];
+        merk.apply(&batch, &[])
+            .map_err(|e| Error::CorruptedData(e.to_string()))
     }
 }
 

grovedb/src/tests.rs

@@ -239,3 +239,79 @@ fn test_root_tree_leafs_are_noted() {
     assert_eq!(db.root_leaf_keys, hm);
     assert_eq!(db.root_tree.leaves_len(), 2);
 }
+
+#[test]
+fn test_checkpoint() {
+    let mut db = make_grovedb();
+    let element1 = Element::Item(b"ayy".to_vec());
+
+    db.insert(&[], b"key1".to_vec(), Element::empty_tree())
+        .expect("cannot insert a subtree 1 into GroveDB");
+    db.insert(&[b"key1"], b"key2".to_vec(), Element::empty_tree())
+        .expect("cannot insert a subtree 2 into GroveDB");
+    db.insert(&[b"key1", b"key2"], b"key3".to_vec(), element1.clone())
+        .expect("cannot insert an item into GroveDB");
+
+    assert_eq!(
+        db.get(&[b"key1", b"key2"], b"key3")
+            .expect("cannot get from grovedb"),
+        element1
+    );
+
+    let checkpoint_tempdir = TempDir::new("checkpoint").expect("cannot open tempdir");
+    let mut checkpoint = db
+        .checkpoint(checkpoint_tempdir.path().join("checkpoint"))
+        .expect("cannot create a checkpoint");
+
+    assert_eq!(
+        db.get(&[b"key1", b"key2"], b"key3")
+            .expect("cannot get from grovedb"),
+        element1
+    );
+    assert_eq!(
+        checkpoint
+            .get(&[b"key1", b"key2"], b"key3")
+            .expect("cannot get from checkpoint"),
+        element1
+    );
+
+    let element2 = Element::Item(b"ayy2".to_vec());
+    let element3 = Element::Item(b"ayy3".to_vec());
+
+    checkpoint
+        .insert(&[b"key1"], b"key4".to_vec(), element2.clone())
+        .expect("cannot insert into checkpoint");
+
+    db.insert(&[b"key1"], b"key4".to_vec(), element3.clone())
+        .expect("cannot insert into GroveDB");
+
+    assert_eq!(
+        checkpoint
+            .get(&[b"key1"], b"key4")
+            .expect("cannot get from checkpoint"),
+        element2,
+    );
+
+    assert_eq!(
+        db.get(&[b"key1"], b"key4")
+            .expect("cannot get from GroveDB"),
+        element3
+    );
+
+    checkpoint
+        .insert(&[b"key1"], b"key5".to_vec(), element3.clone())
+        .expect("cannot insert into checkpoint");
+
+    db.insert(&[b"key1"], b"key6".to_vec(), element3.clone())
+        .expect("cannot insert into GroveDB");
+
+    assert!(matches!(
+        checkpoint.get(&[b"key1"], b"key6"),
+        Err(Error::InvalidPath(_))
+    ));
+
+    assert!(matches!(
+        db.get(&[b"key1"], b"key5"),
+        Err(Error::InvalidPath(_))
+    ));
+}

merk/Cargo.toml

@@ -10,8 +10,9 @@ license = "MIT"
 tempdir = "0.3.7"
 storage = { path = "../storage" }
 thiserror = "1.0.30"
-failure = "0.1.8"
 rocksdb = "0.17.0"
+anyhow = "1.0.51"
+failure = "0.1.8"
 
 [dependencies.time]
 version = "0.1.42"

merk/src/lib.rs

@@ -1,7 +1,5 @@
 #![feature(map_first_last)]
 
-/// Error and Result types.
-mod error;
 /// The top-level store API.
 #[cfg(feature = "full")]
 mod merk;
@@ -18,7 +16,6 @@ pub mod test_utils;
 /// The core tree data structure.
 pub mod tree;
 
-pub use error::{Error, Result};
 #[allow(deprecated)]
 pub use proofs::query::verify_query;
 pub use proofs::query::{execute_proof, verify};

merk/src/merk/chunks.rs

@@ -1,26 +1,21 @@
 //! Provides `ChunkProducer`, which creates chunk proofs for full replication of
 //! a Merk.
+use std::error::Error;
 
-use std::marker::PhantomData;
-
+use anyhow::{anyhow, bail, Result};
 use ed::Encode;
-use failure::bail;
 use storage::{RawIterator, Storage};
 
 use super::Merk;
-use crate::{
-    proofs::{chunk::get_next_chunk, Node, Op},
-    Result,
-};
+use crate::proofs::{chunk::get_next_chunk, Node, Op};
 
 /// A `ChunkProducer` allows the creation of chunk proofs, used for trustlessly
 /// replicating entire Merk trees. Chunks can be generated on the fly in a
 /// random order, or iterated in order for slightly better performance.
 pub struct ChunkProducer<'a, S: Storage + 'a>
 where
     S: Storage,
-    //    crate::error::Error: From<S::Error>,
-    <S as Storage>::Error: std::error::Error + Sync + Send + 'static,
+    <S as Storage>::Error: Error + Sync + Send + 'static,
 {
     trunk: Vec<Op>,
     chunk_boundaries: Vec<Vec<u8>>,
@@ -31,8 +26,7 @@ where
 impl<'a, S> ChunkProducer<'a, S>
 where
     S: Storage,
-    //    crate::error::Error: From<S::Error>,
-    <S as Storage>::Error: std::error::Error + Sync + Send + 'static,
+    <S as Storage>::Error: Error + Sync + Send + 'static,
 {
     /// Creates a new `ChunkProducer` for the given `Merk` instance. In the
     /// constructor, the first chunk (the "trunk") will be created.
@@ -106,7 +100,10 @@ where
                 bail!("Attempted to fetch chunk on empty tree");
             }
             self.index += 1;
-            return self.trunk.encode();
+            return self
+                .trunk
+                .encode()
+                .map_err(|e| anyhow!("cannot get next chunk: {}", e));
         }
 
         if self.index >= self.len() {
@@ -119,15 +116,16 @@ where
         self.index += 1;
 
         let chunk = get_next_chunk(&mut self.raw_iter, end_key_slice)?;
-        chunk.encode()
+        chunk
+            .encode()
+            .map_err(|e| anyhow!("cannot get next chunk: {}", e))
     }
 }
 
 impl<'a, S> IntoIterator for ChunkProducer<'a, S>
 where
     S: Storage,
-    //    crate::error::Error: From<S::Error>,
-    <S as Storage>::Error: std::error::Error + Sync + Send + 'static,
+    <S as Storage>::Error: Error + Sync + Send + 'static,
 {
     type IntoIter = ChunkIter<'a, S>;
     type Item = <ChunkIter<'a, S> as Iterator>::Item;
@@ -143,14 +141,12 @@ where
 pub struct ChunkIter<'a, S>(ChunkProducer<'a, S>)
 where
     S: Storage,
-    //    crate::error::Error: From<S::Error>,
-    <S as Storage>::Error: std::error::Error + Sync + Send + 'static;
+    <S as Storage>::Error: Error + Sync + Send + 'static;
 
 impl<'a, S> Iterator for ChunkIter<'a, S>
 where
     S: Storage,
-    //    crate::error::Error: From<S::Error>,
-    <S as Storage>::Error: std::error::Error + Sync + Send + 'static,
+    <S as Storage>::Error: Error + Sync + Send + 'static,
 {
     type Item = Result<Vec<u8>>;
 
@@ -170,8 +166,7 @@ where
 impl<S> Merk<S>
 where
     S: Storage,
-    //    crate::error::Error: From<S::Error>,
-    <S as Storage>::Error: std::error::Error + Sync + Send + 'static,
+    <S as Storage>::Error: Error + Sync + Send + 'static,
 {
     /// Creates a `ChunkProducer` which can return chunk proofs for replicating
     /// the entire Merk tree.