chore: data format snapshot tests

wnfs-bench/hamt.rs

@@ -5,7 +5,10 @@ use criterion::{
 };
 use proptest::{arbitrary::any, collection::vec, test_runner::TestRunner};
 use std::{cmp, rc::Rc, sync::Arc};
-use wnfs_common::{dagcbor, utils::Sampleable, BlockStore, Link, MemoryBlockStore};
+use wnfs_common::{
+    async_encode, decode, libipld::cbor::DagCborCodec, utils::Sampleable, BlockStore, Link,
+    MemoryBlockStore,
+};
 use wnfs_hamt::{
     diff, merge,
     strategies::{generate_kvs, node_from_kvs, node_from_operations, operations},
@@ -77,7 +80,7 @@ fn node_load_get(c: &mut Criterion) {
             node.set(i.to_string(), i, &store).await.unwrap();
         }
 
-        let encoded_hamt = dagcbor::async_encode(&Hamt::with_root(node), &store)
+        let encoded_hamt = async_encode(&Hamt::with_root(node), &store, DagCborCodec)
             .await
             .unwrap();
 
@@ -87,7 +90,7 @@ fn node_load_get(c: &mut Criterion) {
     c.bench_function("node load and get", |b| {
         b.to_async(AsyncStdExecutor).iter(|| async {
             let encoded_hamt = store.get_deserializable::<Vec<u8>>(&cid).await.unwrap();
-            let hamt: Hamt<String, i32> = dagcbor::decode(encoded_hamt.as_ref()).unwrap();
+            let hamt: Hamt<String, i32> = decode(encoded_hamt.as_ref(), DagCborCodec).unwrap();
 
             for i in 0..50 {
                 assert!(hamt
@@ -109,7 +112,7 @@ fn node_load_remove(c: &mut Criterion) {
             node.set(i.to_string(), i, &store).await.unwrap();
         }
 
-        let encoded_hamt = dagcbor::async_encode(&Hamt::with_root(node), &store)
+        let encoded_hamt = async_encode(&Hamt::with_root(node), &store, DagCborCodec)
             .await
             .unwrap();
 
@@ -120,7 +123,7 @@ fn node_load_remove(c: &mut Criterion) {
         b.to_async(AsyncStdExecutor).iter(|| async {
             let encoded_hamt = store.get_deserializable::<Vec<u8>>(&cid).await.unwrap();
             let mut hamt: Hamt<String, i32> =
-                black_box(dagcbor::decode(encoded_hamt.as_ref()).unwrap());
+                black_box(decode(encoded_hamt.as_ref(), DagCborCodec).unwrap());
 
             for i in 0..50 {
                 let value = hamt.root.remove(&i.to_string(), &store).await.unwrap();
@@ -138,7 +141,7 @@ fn hamt_load_decode(c: &mut Criterion) {
             node.set(i.to_string(), i, &store).await.unwrap();
         }
 
-        let encoded_hamt = dagcbor::async_encode(&Hamt::with_root(node), &store)
+        let encoded_hamt = async_encode(&Hamt::with_root(node), &store, DagCborCodec)
             .await
             .unwrap();
 
@@ -152,7 +155,8 @@ fn hamt_load_decode(c: &mut Criterion) {
     group.bench_function("0", |b| {
         b.to_async(AsyncStdExecutor).iter(|| async {
             let encoded_hamt = store.get_deserializable::<Vec<u8>>(&cid).await.unwrap();
-            let _: Hamt<String, i32> = black_box(dagcbor::decode(encoded_hamt.as_ref()).unwrap());
+            let _: Hamt<String, i32> =
+                black_box(decode(encoded_hamt.as_ref(), DagCborCodec).unwrap());
         })
     });
     group.finish();
@@ -174,7 +178,7 @@ fn hamt_set_encode(c: &mut Criterion) {
 
                 let hamt = Hamt::with_root(node);
 
-                let _ = black_box(dagcbor::async_encode(&hamt, &store).await.unwrap());
+                let _ = black_box(async_encode(&hamt, &store, DagCborCodec).await.unwrap());
             },
             BatchSize::SmallInput,
         )

wnfs-common/Cargo.toml

@@ -20,21 +20,27 @@ authors = ["The Fission Authors"]
 anyhow = "1.0"
 async-once-cell = "0.4"
 async-trait = "0.1"
-bytes = { version = "1.4.0", features = ["serde"] }
-chrono = { version = "0.4.23", default-features = false, features = ["clock", "std"] }
+base64 = { version = "0.21", optional = true }
+base64-serde = { version = "0.7", optional = true }
+bytes = { version = "1.4", features = ["serde"] }
+chrono = { version = "0.4", default-features = false, features = ["clock", "std"] }
 futures = "0.3"
 libipld = { version = "0.16", features = ["dag-cbor", "derive", "serde-codec"] }
 multihash = "0.18"
 once_cell = "1.16"
 proptest = { version = "1.1", optional = true }
 rand_core = "0.6"
 serde = { version = "1.0", features = ["rc"] }
+serde_json = { version = "1.0", optional = true }
 thiserror = "1.0"
 
 [dev-dependencies]
 async-std = { version = "1.11", features = ["attributes"] }
+base64 = "0.21"
+base64-serde = "0.7"
 proptest = "1.1"
 rand = "0.8"
+serde_json = "1.0"
 
 [features]
-test_utils = ["proptest"]
+test_utils = ["dep:proptest", "dep:base64-serde", "dep:base64", "dep:serde_json"]

wnfs-common/src/blockstore.rs

@@ -1,8 +1,9 @@
-use crate::{dagcbor, AsyncSerialize, BlockStoreError, MAX_BLOCK_SIZE};
+use crate::{decode, encode, AsyncSerialize, BlockStoreError, MAX_BLOCK_SIZE};
 use anyhow::{bail, Result};
 use async_trait::async_trait;
 use bytes::Bytes;
 use libipld::{
+    cbor::DagCborCodec,
     cid::Version,
     multihash::{Code, MultihashDigest},
     serde as ipld_serde, Cid,
@@ -50,18 +51,18 @@ pub trait BlockStore: Sized {
 
     async fn get_deserializable<V: DeserializeOwned>(&self, cid: &Cid) -> Result<V> {
         let bytes = self.get_block(cid).await?;
-        let ipld = dagcbor::decode(bytes.as_ref())?;
+        let ipld = decode(bytes.as_ref(), DagCborCodec)?;
         Ok(ipld_serde::from_ipld::<V>(ipld)?)
     }
 
     async fn put_serializable<V: Serialize>(&self, value: &V) -> Result<Cid> {
-        let bytes = dagcbor::encode(&ipld_serde::to_ipld(value)?)?;
+        let bytes = encode(&ipld_serde::to_ipld(value)?, DagCborCodec)?;
         self.put_block(bytes, CODEC_DAG_CBOR).await
     }
 
     async fn put_async_serializable<V: AsyncSerialize>(&self, value: &V) -> Result<Cid> {
         let ipld = value.async_serialize_ipld(self).await?;
-        let bytes = dagcbor::encode(&ipld)?;
+        let bytes = encode(&ipld, DagCborCodec)?;
         self.put_block(bytes, CODEC_DAG_CBOR).await
     }
 
@@ -90,7 +91,11 @@ pub trait BlockStore: Sized {
 ///
 /// IPFS is basically a glorified HashMap.
 #[derive(Debug, Default, Clone, Serialize, Deserialize)]
-pub struct MemoryBlockStore(RefCell<HashMap<String, Bytes>>);
+pub struct MemoryBlockStore(
+    #[serde(serialize_with = "crate::utils::serialize_cid_map")]
+    #[serde(deserialize_with = "crate::utils::deserialize_cid_map")]
+    pub(crate) RefCell<HashMap<Cid, Bytes>>,
+);
 
 impl MemoryBlockStore {
     /// Creates a new in-memory block store.
@@ -106,7 +111,7 @@ impl BlockStore for MemoryBlockStore {
         let bytes = self
             .0
             .borrow()
-            .get(&cid.to_string())
+            .get(cid)
             .ok_or(BlockStoreError::CIDNotFound(*cid))?
             .clone();
 
@@ -122,7 +127,7 @@ impl BlockStore for MemoryBlockStore {
         let cid = self.create_cid(&bytes, codec)?;
 
         // Insert the bytes into the HashMap using the CID as the key
-        self.0.borrow_mut().insert(cid.to_string(), bytes);
+        self.0.borrow_mut().insert(cid, bytes);
 
         // Return Ok status with the generated CID
         Ok(cid)
@@ -192,9 +197,9 @@ pub async fn bs_serialization_test<
     // Insert the object into the blockstore
     let cid = store.put_serializable(&bytes).await?;
     // Serialize the BlockStore
-    let serial_store: Vec<u8> = dagcbor::encode(&store)?;
+    let serial_store: Vec<u8> = encode(&store, DagCborCodec)?;
     // Construct a new BlockStore from the Serialized object
-    let deserial_store: T = dagcbor::decode(&serial_store)?;
+    let deserial_store: T = decode(&serial_store, DagCborCodec)?;
     // Retrieve the object from the blockstore
     let loaded: Vec<u8> = deserial_store.get_deserializable(&cid).await?;
     // Assert that the objects are the same as the ones we inserted

wnfs-common/src/encoding.rs

@@ -1,37 +1,46 @@
-/// Helper methods for decoding and encoding values into DagCbor.
-pub mod dagcbor {
-    use crate::{AsyncSerialize, BlockStore};
-    use anyhow::Result;
-    use libipld::{
-        cbor::DagCborCodec,
-        codec::{Decode, Encode},
-        serde as ipld_serde, Ipld,
-    };
-    use serde::{de::DeserializeOwned, Serialize};
-    use std::io::Cursor;
+use crate::{AsyncSerialize, BlockStore};
+use anyhow::Result;
+use libipld::{
+    codec::{Decode, Encode},
+    prelude::Codec,
+    serde as ipld_serde, Ipld,
+};
+use serde::{de::DeserializeOwned, Serialize};
+use std::io::Cursor;
 
-    /// Encodes a serializable value into DagCbor bytes.
-    pub fn encode<S: Serialize>(value: &S) -> Result<Vec<u8>> {
-        let ipld = ipld_serde::to_ipld(value)?;
-        let mut bytes = Vec::new();
-        ipld.encode(DagCborCodec, &mut bytes)?;
-        Ok(bytes)
-    }
+/// Encodes a serializable value into DagCbor bytes.
+pub fn encode<S, C>(value: &S, codec: C) -> Result<Vec<u8>>
+where
+    S: Serialize,
+    C: Codec,
+    Ipld: Encode<C>,
+{
+    let ipld = ipld_serde::to_ipld(value)?;
+    let mut bytes = Vec::new();
+    <Ipld as Encode<C>>::encode(&ipld, codec, &mut bytes)?;
+    Ok(bytes)
+}
 
-    /// Encodes an async serializable value into DagCbor bytes.
-    pub async fn async_encode<V: AsyncSerialize>(
-        value: &V,
-        store: &impl BlockStore,
-    ) -> Result<Vec<u8>> {
-        let ipld = value.async_serialize_ipld(store).await?;
-        let mut bytes = Vec::new();
-        ipld.encode(DagCborCodec, &mut bytes)?;
-        Ok(bytes)
-    }
+/// Encodes an async serializable value into DagCbor bytes.
+pub async fn async_encode<V, C>(value: &V, store: &impl BlockStore, codec: C) -> Result<Vec<u8>>
+where
+    V: AsyncSerialize,
+    C: Codec,
+    Ipld: Encode<C>,
+{
+    let ipld = value.async_serialize_ipld(store).await?;
+    let mut bytes = Vec::new();
+    <Ipld as Encode<C>>::encode(&ipld, codec, &mut bytes)?;
+    Ok(bytes)
+}
 
-    /// Decodes recieved DagCbor bytes into a deserializable value.
-    pub fn decode<D: DeserializeOwned>(bytes: &[u8]) -> Result<D> {
-        let ipld = Ipld::decode(DagCborCodec, &mut Cursor::new(bytes))?;
-        Ok(ipld_serde::from_ipld::<_>(ipld)?)
-    }
+/// Decodes recieved DagCbor bytes into a deserializable value.
+pub fn decode<D, C>(bytes: &[u8], codec: C) -> Result<D>
+where
+    D: DeserializeOwned,
+    C: Codec,
+    Ipld: Decode<C>,
+{
+    let ipld = <Ipld as Decode<C>>::decode(codec, &mut Cursor::new(bytes))?;
+    Ok(ipld_serde::from_ipld::<_>(ipld)?)
 }

wnfs-common/src/error.rs

@@ -16,6 +16,9 @@ pub enum BlockStoreError {
     #[error("Cannot find specified CID in block store: {0}")]
     CIDNotFound(Cid),
 
+    #[error("Cannot find handler for block with CID: {0}")]
+    BlockHandlerNotFound(Cid),
+
     #[error("Lock poisoned")]
     LockPoisoned,
 }

wnfs-common/src/lib.rs

@@ -30,3 +30,11 @@ pub const MAX_BLOCK_SIZE: usize = usize::pow(2, 18);
 
 /// The general size of digests in WNFS.
 pub type HashOutput = [u8; HASH_BYTE_SIZE];
+
+//--------------------------------------------------------------------------------------------------
+// Re-exports
+//--------------------------------------------------------------------------------------------------
+
+pub mod libipld {
+    pub use libipld::*;
+}

wnfs-common/src/metadata.rs

@@ -270,15 +270,16 @@ impl<'de> Deserialize<'de> for NodeType {
 
 #[cfg(test)]
 mod tests {
-    use crate::{dagcbor, Metadata};
+    use crate::{decode, encode, Metadata};
     use chrono::Utc;
+    use libipld::cbor::DagCborCodec;
 
     #[async_std::test]
     async fn metadata_can_encode_decode_as_cbor() {
         let metadata = Metadata::new(Utc::now());
 
-        let encoded_metadata = dagcbor::encode(&metadata).unwrap();
-        let decoded_metadata = dagcbor::decode::<Metadata>(encoded_metadata.as_ref()).unwrap();
+        let encoded_metadata = encode(&metadata, DagCborCodec).unwrap();
+        let decoded_metadata: Metadata = decode(encoded_metadata.as_ref(), DagCborCodec).unwrap();
 
         assert_eq!(metadata, decoded_metadata);
     }