feat: Switch from Namefilter to Name Accumulators (#247)

* feat: Add wnfs-nameaccumulator crate * WIP: refactor * WIP * Type checker * Checkpoint: No failing unit tests * Remove wnfs-namefilter * feat: Benchmarks for name accumulators * chore: Use published skip ratchet version * refactor out Name abstraction * cargo clippy --fix * Run proptests less often * refactor: use shorter AccumulatorSetup names * Make header's name based on mountpoint * Mount files and directories to explicit paths * refactor: Some forest function renames & moving code around * Implement proof generation * Attach proof info to PrivateForest entries * One proof for each CID in forest * Implement batch proving library functions * Revert PrivateForest back to storing just sets of CIDs * Refactor out trait PrivateForest * Refactor PrivateForest trait to be Rc-independent * Move forest-related files into own directory * Don't re-expose, implement PrivateForest with and without Rc for HamtForest * Implement get_mut in hamt node * Implement private forest label proof aggregation * Fix mounting * Write an example for forest proofs * Fix some clippy warnings * Fix doctests * Move more structs into serializable.rs * Fix all clippy warnings. * Use once_cell crate instead of std::cell for backwards compat * Make clippy happy? * Make clippy nightly happy! * Possibly fix wasm bindings * Uncomment two tests * Also prove removals in proving forest * Go through some TODOs * Zeroize toxic waste * Consistently use `CryptoRngCore` instead of `RngCore` * Write docstrings * Document more & remove unused * Organize name accumulator tests * Move proofs example to example dir * Fix wasm types * Tests in proofs.rs * Small cleanup and fixes * Switch to big endian for BigUints in general (RSA moduli are usually encoded big-endian) * Update readme, remove `Namefilter` mentions * Canonicalize l hash derivation * Correctly encode big integers with padding * Add previously failing test case * Better domain separation strings * Derive the name segment directly from the key * Remove redundant `as_ref()` + format * Remove `with_seed` and related APIs * Use `HamtForest::load` and `store` everywhere * Remove generic from `HamtForest` * Update wnfs/src/private/keys/privateref.rs Co-authored-by: Stephen Akinyemi <[email protected]> Signed-off-by: Philipp Krüger <[email protected]> * Update wnfs/examples/write_proofs.rs Co-authored-by: Stephen Akinyemi <[email protected]> Signed-off-by: Philipp Krüger <[email protected]> * Rename "saturated" into "revisioned" * Run webpack as a separate CI step * Adjust new js test from main --------- Signed-off-by: Philipp Krüger <[email protected]> Co-authored-by: Stephen Akinyemi <[email protected]>
wnfs-wg · Jul 7, 2023 · 7026a37 · 7026a37
1 parent 982feff
commit 7026a37
Show file tree

Hide file tree

Showing 69 changed files with 3,508 additions and 2,451 deletions.
diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml
@@ -127,5 +127,8 @@ jobs:
       - name: Install Playwright
         run: npx playwright install --with-deps
 
+      - name: Run Webpack
+        run: cd tests/server && npx webpack
+
       - name: Run Playwright Tests
         run: yarn playwright test
diff --git a/.release-please-manifest.json b/.release-please-manifest.json
@@ -3,6 +3,6 @@
   "wnfs-bench": "0.1.22",
   "wnfs-common": "0.1.22",
   "wnfs-hamt": "0.1.22",
-  "wnfs-namefilter": "0.1.22",
+  "wnfs-nameaccumulator": "0.1.22",
   "wnfs-wasm": "0.1.22"
 }
diff --git a/Cargo.toml b/Cargo.toml
@@ -4,7 +4,7 @@ members = [
 	"wnfs-bench",
 	"wnfs-common",
 	"wnfs-hamt",
-	"wnfs-namefilter",
+	"wnfs-nameaccumulator",
 	"wnfs-wasm",
 ]
 

diff --git a/README.md b/README.md
@@ -55,7 +55,7 @@ This library is designed with WebAssembly in mind. You can follow instructions o
 - [wnfs](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs)
 - [wnfs-common](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-common)
 - [wnfs-hamt](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-hamt)
-- [wnfs-namefilter](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-namefilter)
+- [wnfs-nameaccumulator](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-nameaccumulator)
 - [wnfs-wasm](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-wasm)
 
 ## Building the Project
@@ -216,9 +216,11 @@ use anyhow::Result;
 use chrono::Utc;
 use rand::thread_rng;
 use std::rc::Rc;
-use wnfs::private::{PrivateDirectory, PrivateForest};
+use wnfs::private::{
+    PrivateDirectory,
+    forest::{hamt::HamtForest, traits::PrivateForest},
+};
 use wnfs_common::MemoryBlockStore;
-use wnfs_namefilter::Namefilter;
 
 #[async_std::main]
 async fn main() -> Result<()> {
@@ -229,11 +231,11 @@ async fn main() -> Result<()> {
     let rng = &mut thread_rng();
 
     // Create a private forest.
-    let forest = &mut Rc::new(PrivateForest::new());
+    let forest = &mut Rc::new(HamtForest::new_trusted(rng));
 
     // Create a new private directory.
     let dir = &mut Rc::new(PrivateDirectory::new(
-        Namefilter::default(),
+        &forest.empty_name(),
         Utc::now(),
         rng,
     ));
@@ -270,11 +272,11 @@ async fn main() -> Result<()> {
 }
 ```
 
-This example introduces a few new concepts. The first is the `PrivateForest` which is a HAMT that can contain multiple file trees.
+This example introduces a few new concepts. The first is the `HamtForest` which is a HAMT that can contain multiple file trees and implements the `PrivateForest` interface needed for persisting private file systems.
 
-The second is the `Namefilter` (a fixed-size bloomfilter) that lets us identify nodes in the filesystem, and are suitable for offspring checks. Namefilters currently have limitation on how deep the file tree can go but that is going to change in the near future.
+The second is the `Name` (returned from `forest.empty_name()`) and `NameAccumulator` that lets us identify nodes in the filesystem, and are suitable for offspring proving.
 
-Finally, we have the random number generator, `rng`, that the library uses for ridding predictability and avoiding collisions in the `PrivateForest`.
+Finally, we have the random number generator, `rng`, that the library uses for generating new keys and other random values needed for the protocol.
 
 Check the [`wnfs/examples/`][wnfs-examples] folder for more examples.
 

diff --git a/clippy.toml b/clippy.toml
@@ -1 +1,6 @@
 cognitive-complexity-threshold = 10
+ignore-interior-mutability = [
+    "bytes::Bytes",
+    "wnfs::private::encrypted::Encrypted",
+    "wnfs_nameaccumulator::NameAccumulator",
+]
diff --git a/release-please-config.json b/release-please-config.json
@@ -9,6 +9,6 @@
     "wnfs-wasm": {},
     "wnfs-common": {},
     "wnfs-hamt": {},
-    "wnfs-namefilter": {}
+    "wnfs-nameaccumulator": {}
   }
 }
diff --git a/scripts/rs-wnfs.sh b/scripts/rs-wnfs.sh
@@ -73,8 +73,8 @@ help() {
     echo "    rs-wnfs [COMMAND] [...args]"
     echo ""
     echo "COMMAND:"
-    echo "   * build [--fs|--wasm|--common|--hamt|--filter]  - build projects"
-    echo "   * test  [--fs|--wasm|--common|--hamt|--filter]  - run tests"
+    echo "   * build [--fs|--wasm|--common|--hamt|--accumulator]  - build projects"
+    echo "   * test  [--fs|--wasm|--common|--hamt|--accumulator]  - run tests"
     echo "   * bench                                             - run wnfs benchmarks"
     echo "   * setup                                             - install rs-wnfs script"
     echo "   * help                                              - print this help message"
@@ -90,7 +90,7 @@ help() {
 #	Builds the project.
 #
 # USAGE:
-#	rs-wnfs build [--fs|--wasm|--common|--hamt|--filter]
+#	rs-wnfs build [--fs|--wasm|--common|--hamt|--accumulator]
 #
 build() {
 	if check_flag --fs; then
@@ -101,12 +101,12 @@ build() {
         build_common
     elif check_flag --hamt; then
         build_hamt
-    elif check_flag --filter; then
-        build_filter
+    elif check_flag --accumulator; then
+        build_accumulator
     else
         build_common
         build_hamt
-        build_filter
+        build_accumulator
         build_fs
         build_wasm
     fi
@@ -127,9 +127,9 @@ build_hamt() {
     cargo build -p wnfs-hamt --release
 }
 
-build_filter() {
-    display_header "💿 | BUILDING WNFS-NAMEFILTER PROJECT | 💿"
-    cargo build -p wnfs-namefilter --release
+build_accumulator() {
+    display_header "💿 | BUILDING WNFS-NAMEACCUMULATOR PROJECT | 💿"
+    cargo build -p wnfs-nameaccumulator --release
 }
 
 build_wasm() {
@@ -147,7 +147,7 @@ build_wasm() {
 #   Runs tests.
 #
 # USAGE:
-#	rs-wnfs test [--fs|--wasm|--common|--hamt|--filter]
+#	rs-wnfs test [--fs|--wasm|--common|--hamt|--accumulator]
 #
 test() {
 	if check_flag --fs; then
@@ -158,12 +158,12 @@ test() {
         test_common
     elif check_flag --hamt; then
         test_hamt
-    elif check_flag --filter; then
-        test_filter
+    elif check_flag --accumulator; then
+        test_accumulator
     else
         test_common
         test_hamt
-        test_filter
+        test_accumulator
         test_fs
         test_wasm
     fi
@@ -184,9 +184,9 @@ test_hamt() {
     cargo test -p wnfs-hamt
 }
 
-test_filter() {
-    display_header "🧪 | RUNNING WNFS-NAMEFILTER TESTS | 🧪"
-    cargo test -p wnfs-namefilter
+test_accumulator() {
+    display_header "🧪 | RUNNING WNFS-NAMEACCUMULATOR TESTS | 🧪"
+    cargo test -p wnfs-nameaccumulator
 }
 
 test_wasm() {

diff --git a/wnfs-bench/Cargo.toml b/wnfs-bench/Cargo.toml
@@ -11,17 +11,19 @@ async-std = { version = "1.11", features = ["attributes"] }
 chrono = { version = "0.4", default-features = false, features = ["clock", "std"] }
 criterion = { version = "0.4", features = ["async_std"] }
 proptest = "1.1"
+rand = "0.8"
+sha3 = "0.10"
 wnfs = { path = "../wnfs" }
 wnfs-common = { path = "../wnfs-common", features = ["test_utils"] }
 wnfs-hamt = { path = "../wnfs-hamt", features = ["test_utils"] }
-wnfs-namefilter = { path = "../wnfs-namefilter" }
+wnfs-nameaccumulator = { path = "../wnfs-nameaccumulator" }
 
 [[bench]]
 name = "hamt"
 harness = false
 path = "hamt.rs"
 
 [[bench]]
-name = "namefilter"
+name = "nameaccumulator"
 harness = false
-path = "namefilter.rs"
+path = "nameaccumulator.rs"
diff --git a/wnfs-bench/hamt.rs b/wnfs-bench/hamt.rs
@@ -14,10 +14,10 @@ use wnfs_hamt::{
 
 fn node_set(c: &mut Criterion) {
     let mut runner = TestRunner::deterministic();
-    let mut store = MemoryBlockStore::default();
+    let store = MemoryBlockStore::default();
     let operations = operations(any::<[u8; 32]>(), any::<u64>(), 1_000_000).sample(&mut runner);
     let node =
-        &async_std::task::block_on(async { node_from_operations(&operations, &mut store).await })
+        &async_std::task::block_on(async { node_from_operations(&operations, &store).await })
             .expect("Couldn't setup HAMT node from operations");
 
     let store = Arc::new(store);
@@ -30,12 +30,11 @@ fn node_set(c: &mut Criterion) {
                 (store, kv)
             },
             |(store, (key, value))| async move {
-                black_box(
-                    Rc::clone(node)
-                        .set(key, value, store.as_ref())
-                        .await
-                        .unwrap(),
-                );
+                Rc::clone(node)
+                    .set(key, value, store.as_ref())
+                    .await
+                    .unwrap();
+                black_box(());
             },
             BatchSize::SmallInput,
         );
@@ -48,11 +47,11 @@ fn node_set_consecutive(c: &mut Criterion) {
     c.bench_function("node set 1000 consecutive", |b| {
         b.to_async(AsyncStdExecutor).iter_batched(
             || {
-                let mut store = MemoryBlockStore::default();
+                let store = MemoryBlockStore::default();
                 let operations =
                     operations(any::<[u8; 32]>(), any::<u64>(), 1000).sample(&mut runner);
                 let node = async_std::task::block_on(async {
-                    node_from_operations(&operations, &mut store).await
+                    node_from_operations(&operations, &store).await
                 })
                 .expect("Couldn't setup HAMT node from operations");
 
@@ -61,7 +60,8 @@ fn node_set_consecutive(c: &mut Criterion) {
             },
             |(mut node, store, kvs)| async move {
                 for (key, value) in kvs {
-                    black_box(node.set(key, value, &store).await.unwrap());
+                    node.set(key, value, &store).await.unwrap();
+                    black_box(());
                 }
             },
             BatchSize::SmallInput,
@@ -70,14 +70,14 @@ fn node_set_consecutive(c: &mut Criterion) {
 }
 
 fn node_load_get(c: &mut Criterion) {
-    let mut store = MemoryBlockStore::default();
+    let store = MemoryBlockStore::default();
     let cid = async_std::task::block_on(async {
         let mut node = Rc::new(<Node<_, _>>::default());
         for i in 0..50 {
             node.set(i.to_string(), i, &store).await.unwrap();
         }
 
-        let encoded_hamt = dagcbor::async_encode(&Hamt::with_root(node), &mut store)
+        let encoded_hamt = dagcbor::async_encode(&Hamt::with_root(node), &store)
             .await
             .unwrap();
 
@@ -102,14 +102,14 @@ fn node_load_get(c: &mut Criterion) {
 }
 
 fn node_load_remove(c: &mut Criterion) {
-    let mut store = MemoryBlockStore::default();
+    let store = MemoryBlockStore::default();
     let cid = async_std::task::block_on(async {
         let mut node = Rc::new(<Node<_, _>>::default());
         for i in 0..50 {
             node.set(i.to_string(), i, &store).await.unwrap();
         }
 
-        let encoded_hamt = dagcbor::async_encode(&Hamt::with_root(node), &mut store)
+        let encoded_hamt = dagcbor::async_encode(&Hamt::with_root(node), &store)
             .await
             .unwrap();
 
@@ -131,14 +131,14 @@ fn node_load_remove(c: &mut Criterion) {
 }
 
 fn hamt_load_decode(c: &mut Criterion) {
-    let mut store = MemoryBlockStore::default();
+    let store = MemoryBlockStore::default();
     let (cid, bytes) = async_std::task::block_on(async {
         let mut node = Rc::new(<Node<_, _>>::default());
         for i in 0..50 {
             node.set(i.to_string(), i, &store).await.unwrap();
         }
 
-        let encoded_hamt = dagcbor::async_encode(&Hamt::with_root(node), &mut store)
+        let encoded_hamt = dagcbor::async_encode(&Hamt::with_root(node), &store)
             .await
             .unwrap();
 
@@ -167,14 +167,14 @@ fn hamt_set_encode(c: &mut Criterion) {
                     Rc::new(<Node<_, _>>::default()),
                 )
             },
-            |(mut store, mut node)| async move {
+            |(store, mut node)| async move {
                 for i in 0..50 {
                     node.set(i.to_string(), i, &store).await.unwrap();
                 }
 
                 let hamt = Hamt::with_root(node);
 
-                let _ = black_box(dagcbor::async_encode(&hamt, &mut store).await.unwrap());
+                let _ = black_box(dagcbor::async_encode(&hamt, &store).await.unwrap());
             },
             BatchSize::SmallInput,
         )
@@ -187,20 +187,20 @@ fn hamt_diff(c: &mut Criterion) {
     c.bench_function("hamt diff", |b| {
         b.to_async(AsyncStdExecutor).iter_batched(
             || {
-                let mut store = MemoryBlockStore::default();
+                let store = MemoryBlockStore::default();
                 let kvs1 = generate_kvs("[a-z0-9]{1,3}", 0u64..1000, 0..100).sample(&mut runner);
                 let kvs2 = generate_kvs("[a-z0-9]{1,3}", 0u64..1000, 0..100).sample(&mut runner);
                 let (node1, node2) = task::block_on(async {
                     (
-                        node_from_kvs(kvs1, &mut store).await.unwrap(),
-                        node_from_kvs(kvs2, &mut store).await.unwrap(),
+                        node_from_kvs(kvs1, &store).await.unwrap(),
+                        node_from_kvs(kvs2, &store).await.unwrap(),
                     )
                 });
                 (store, (node1, node2))
             },
-            |(mut store, (node1, node2))| async move {
+            |(store, (node1, node2))| async move {
                 black_box(
-                    diff(Link::from(node1), Link::from(node2), &mut store)
+                    diff(Link::from(node1), Link::from(node2), &store)
                         .await
                         .unwrap(),
                 );
@@ -216,24 +216,24 @@ fn hamt_merge(c: &mut Criterion) {
     c.bench_function("hamt merge", |b| {
         b.to_async(AsyncStdExecutor).iter_batched(
             || {
-                let mut store = MemoryBlockStore::default();
+                let store = MemoryBlockStore::default();
                 let kvs1 = generate_kvs("[a-z0-9]{1,3}", 0u64..1000, 0..100).sample(&mut runner);
                 let kvs2 = generate_kvs("[a-z0-9]{1,3}", 0u64..1000, 0..100).sample(&mut runner);
                 let (node1, node2) = task::block_on(async {
                     (
-                        node_from_kvs(kvs1, &mut store).await.unwrap(),
-                        node_from_kvs(kvs2, &mut store).await.unwrap(),
+                        node_from_kvs(kvs1, &store).await.unwrap(),
+                        node_from_kvs(kvs2, &store).await.unwrap(),
                     )
                 });
                 (store, (node1, node2))
             },
-            |(mut store, (node1, node2))| async move {
+            |(store, (node1, node2))| async move {
                 black_box(
                     merge(
                         Link::from(node1),
                         Link::from(node2),
                         |a, b| Ok(cmp::min(*a, *b)),
-                        &mut store,
+                        &store,
                     )
                     .await
                     .unwrap(),