Private File Sharding

.github/workflows/checks.yaml

@@ -80,15 +80,15 @@ jobs:
       - name: Run Tests
         run: cargo test --all-features
 
-
   wasm-js-tests:
     strategy:
       fail-fast: false
       matrix:
         rust-toolchain:
           - stable
           - nightly
-    runs-on: ubuntu-latest
+
+    runs-on: macos-latest
 
     defaults:
       run:

README.md

@@ -315,7 +315,7 @@ We would be happy to try to answer your question or try opening a new issue on G
 This project is licensed under the [Apache License 2.0](https://github.com/wnfs-wg/rs-wnfs/blob/main/LICENSE).
 
 [benchmarks]: https://wnfs-wg.github.io/rs-wnfs/dev/bench/
-[blockstore-trait]: wnfs/common/blockstore.rs#L30-L86
+[blockstore-trait]: https://github.com/wnfs-wg/rs-wnfs/blob/main/wnfs/src/common/blockstore.rs
 [hamt-wiki]: https://en.wikipedia.org/wiki/Hash_array_mapped_trie
 [ipld-spec]: https://ipld.io/
 [npm-ipld-tools]: https://www.npmjs.com/search?q=ipld

wnfs-bench/Cargo.toml

@@ -2,27 +2,13 @@
 name = "wnfs-bench"
 version = "0.1.9"
 description = "WNFS Benchmarks"
-keywords = ["wnfs", "webnative", "ipfs", "decentralisation", "benchmarks"]
-categories = ["filesystem", "cryptography", "web-programming", "wasm"]
 edition = "2021"
-repository = "https://github.com/wnfs-wg/rs-wnfs/tree/main/bench"
-homepage = "https://fission.codes"
-authors = ["The Fission Authors"]
-
-[dependencies]
-async-std = { version = "1.11", features = ["attributes"] }
-proptest = "1.0"
-wnfs = { path = "../wnfs", optional = true }
 
 [dev-dependencies]
+async-std = { version = "1.11", features = ["attributes"] }
 criterion = { version = "0.4", features = ["async_std"] }
-
-[lib]
-path = "lib.rs"
-bench = false
-
-[features]
-default = ["wnfs/test_strategies"]
+proptest = "1.0"
+wnfs = { path = "../wnfs", features = ["test_strategies"] }
 
 [[bench]]
 name = "hamt"

wnfs-wasm/README.md

@@ -170,7 +170,7 @@ console.log("Files in /pictures directory:", result);
   npm publish
   ```
 
-[blockstore-trait]: wnfs/common/blockstore.rs#L30-L86
+[blockstore-trait]: https://github.com/wnfs-wg/rs-wnfs/blob/main/wnfs/src/common/blockstore.rs
 [hamt-wiki]: https://en.wikipedia.org/wiki/Hash_array_mapped_trie
 [ipld-spec]: https://ipld.io/
 [npm-ipld-tools]: https://www.npmjs.com/search?q=ipld

wnfs-wasm/playwright.config.ts

@@ -55,6 +55,12 @@ const config: PlaywrightTestConfig = {
         ...devices["Desktop Firefox"],
       },
     },
+    {
+      name: "safari",
+      use: {
+        ...devices["Desktop Safari"],
+      },
+    },
   ],
 
   /* Folder for test artifacts such as screenshots, videos, traces, etc. */

wnfs-wasm/src/fs/private/file.rs

@@ -1,42 +1,91 @@
 //! The bindgen API for PrivateFile.
 
+use crate::{
+    fs::{
+        utils::{self, error},
+        BlockStore, ForeignBlockStore, JsResult, Namefilter, PrivateForest, Rng,
+    },
+    value,
+};
 use chrono::{DateTime, Utc};
-use js_sys::Date;
-use wasm_bindgen::prelude::wasm_bindgen;
+use js_sys::{Date, Promise, Uint8Array};
+use std::rc::Rc;
+use wasm_bindgen::{prelude::wasm_bindgen, JsValue};
+use wasm_bindgen_futures::future_to_promise;
 use wnfs::{Id, PrivateFile as WnfsPrivateFile};
 
-use crate::fs::{JsResult, Namefilter, Rng};
-
 //--------------------------------------------------------------------------------------------------
 // Type Definitions
 //--------------------------------------------------------------------------------------------------
 
 /// A file in a WNFS public file system.
 #[wasm_bindgen]
-pub struct PrivateFile(WnfsPrivateFile);
+pub struct PrivateFile(Rc<WnfsPrivateFile>);
 
 //--------------------------------------------------------------------------------------------------
 // Implementations
 //--------------------------------------------------------------------------------------------------
 
 #[wasm_bindgen]
 impl PrivateFile {
-    /// Creates a new private file.
+    /// Creates an empty private file.
     #[wasm_bindgen(constructor)]
-    pub fn new(
+    pub fn new(parent_bare_name: Namefilter, time: &Date, mut rng: Rng) -> JsResult<PrivateFile> {
+        let time = DateTime::<Utc>::from(time);
+
+        Ok(Self(Rc::new(WnfsPrivateFile::new(
+            parent_bare_name.0,
+            time,
+            &mut rng,
+        ))))
+    }
+
+    /// Creates a file with provided content.
+    #[wasm_bindgen(js_name = "withContent")]
+    pub fn with_content(
         parent_bare_name: Namefilter,
         time: &Date,
         content: Vec<u8>,
+        hamt: PrivateForest,
+        store: BlockStore,
         mut rng: Rng,
-    ) -> JsResult<PrivateFile> {
+    ) -> JsResult<Promise> {
+        let mut store = ForeignBlockStore(store);
         let time = DateTime::<Utc>::from(time);
 
-        Ok(PrivateFile(WnfsPrivateFile::new(
-            parent_bare_name.0,
-            time,
-            content,
-            &mut rng,
-        )))
+        Ok(future_to_promise(async move {
+            let (file, hamt) = WnfsPrivateFile::with_content(
+                parent_bare_name.0,
+                time,
+                content,
+                hamt.0,
+                &mut store,
+                &mut rng,
+            )
+            .await
+            .map_err(error("Cannot create a file with provided content"))?;
+
+            Ok(utils::create_private_file_result(
+                PrivateFile(Rc::new(file)),
+                PrivateForest(hamt),
+            )?)
+        }))
+    }
+
+    /// Gets the entire content of a file.
+    #[wasm_bindgen(js_name = "getContent")]
+    pub fn get_content(&self, hamt: PrivateForest, store: BlockStore) -> JsResult<Promise> {
+        let file = Rc::clone(&self.0);
+        let store = ForeignBlockStore(store);
+
+        Ok(future_to_promise(async move {
+            let content = file
+                .get_content(&hamt.0, &store)
+                .await
+                .map_err(error("Cannot get content of file"))?;
+
+            Ok(value!(Uint8Array::from(content.as_slice())))
+        }))
     }
 
     /// Gets a unique id for node.

wnfs-wasm/src/fs/utils.rs

@@ -9,12 +9,19 @@ use wnfs::{
     Metadata,
 };
 
-use super::{metadata::JsMetadata, PrivateDirectory, PrivateForest, PublicDirectory};
+use super::{metadata::JsMetadata, PrivateDirectory, PrivateFile, PrivateForest, PublicDirectory};
 
 //--------------------------------------------------------------------------------------------------
 // Functions
 //--------------------------------------------------------------------------------------------------
 
+pub(crate) fn error<E>(message: &str) -> impl FnOnce(E) -> js_sys::Error + '_
+where
+    E: Debug,
+{
+    move |e| Error::new(&format!("{message}: {e:?}"))
+}
+
 pub(crate) fn map_to_rust_vec<T, F: FnMut(JsValue) -> JsResult<T>>(
     array: &Array,
     f: F,
@@ -43,8 +50,10 @@ pub(crate) fn create_public_op_result<T: Into<JsValue>>(
         &op_result,
         &value!("rootDir"),
         &PublicDirectory(root_dir).into(),
-    )?;
-    Reflect::set(&op_result, &value!("result"), &result.into())?;
+    )
+    .map_err(error("Failed to set rootDir"))?;
+    Reflect::set(&op_result, &value!("result"), &result.into())
+        .map_err(error("Failed to set result"))?;
 
     Ok(value!(op_result))
 }
@@ -54,35 +63,44 @@ pub(crate) fn create_private_op_result<T: Into<JsValue>>(
     hamt: Rc<WnfsPrivateForest>,
     result: T,
 ) -> JsResult<JsValue> {
-    let op_result = Object::new();
+    let op_result = Array::new();
 
     Reflect::set(
         &op_result,
         &value!("rootDir"),
         &PrivateDirectory(root_dir).into(),
-    )?;
-    Reflect::set(&op_result, &value!("hamt"), &PrivateForest(hamt).into())?;
-    Reflect::set(&op_result, &value!("result"), &result.into())?;
+    )
+    .map_err(error("Failed to set rootDir"))?;
+    Reflect::set(&op_result, &value!("hamt"), &PrivateForest(hamt).into())
+        .map_err(error("Failed to set hamt"))?;
+    Reflect::set(&op_result, &value!("result"), &result.into())
+        .map_err(error("Failed to set result"))?;
 
     Ok(value!(op_result))
 }
 
-pub(crate) fn error<E>(message: &str) -> impl FnOnce(E) -> js_sys::Error + '_
-where
-    E: Debug,
-{
-    move |e| Error::new(&format!("{message}: {e:?}"))
+pub(crate) fn create_private_file_result(
+    file: PrivateFile,
+    hamt: PrivateForest,
+) -> JsResult<JsValue> {
+    let op_result = Array::new();
+
+    Reflect::set(&op_result, &value!(0), &file.into()).map_err(error("Failed to set file"))?;
+    Reflect::set(&op_result, &value!(1), &hamt.into()).map_err(error("Failed to set hamt"))?;
+
+    Ok(value!(op_result))
 }
 
 pub(crate) fn create_ls_entry(name: &String, metadata: &Metadata) -> JsResult<JsValue> {
     let entry = Object::new();
 
-    Reflect::set(&entry, &value!("name"), &value!(name))?;
+    Reflect::set(&entry, &value!("name"), &value!(name)).map_err(error("Failed to set name"))?;
     Reflect::set(
         &entry,
         &value!("metadata"),
         &JsMetadata(metadata).try_into()?,
-    )?;
+    )
+    .map_err(error("Failed to set metadata"))?;
 
     Ok(value!(entry))
 }

wnfs-wasm/tests/private.spec.ts

@@ -339,3 +339,74 @@ test.describe("PrivateDirectory", () => {
     expect(picturesContent[0].name).toEqual("cats");
   });
 });
+
+test.describe("PrivateFile", () => {
+  test("empty can create empty file", async ({ page }) => {
+    const result = await page.evaluate(async () => {
+      const {
+        wnfs: { PrivateFile, Namefilter },
+        mock: { Rng },
+      } = await window.setup();
+
+      const rng = new Rng();
+      const file = new PrivateFile(new Namefilter(), new Date(), rng);
+
+      return file.getId();
+    });
+
+    expect(result).toBeDefined();
+  });
+
+  test("withContent can create file with content", async ({ page }) => {
+    const result = await page.evaluate(async () => {
+      const {
+        wnfs: { PrivateFile, PrivateForest, Namefilter },
+        mock: { MemoryBlockStore, Rng },
+      } = await window.setup();
+
+      const hamt = new PrivateForest();
+      const rng = new Rng();
+      const store = new MemoryBlockStore();
+      const [file] = await PrivateFile.withContent(
+        new Namefilter(),
+        new Date(),
+        new Uint8Array([1, 2, 3, 4, 5]),
+        hamt,
+        store,
+        rng
+      );
+
+      return file.getId();
+    });
+
+    expect(result).toBeDefined();
+  });
+
+  test("getContent can fetch file's entire content", async ({ page }) => {
+    const [length, type] = await page.evaluate(async () => {
+      const {
+        wnfs: { PrivateFile, PrivateForest, Namefilter },
+        mock: { MemoryBlockStore, Rng },
+      } = await window.setup();
+
+      const initialHamt = new PrivateForest();
+      const rng = new Rng();
+      const store = new MemoryBlockStore();
+      var [file, hamt] = await PrivateFile.withContent(
+        new Namefilter(),
+        new Date(),
+        new Uint8Array([1, 2, 3, 4, 5]),
+        initialHamt,
+        store,
+        rng
+      );
+
+      let content = await file.getContent(hamt, store);
+
+      return [content.length, content.constructor.name, content];
+    });
+
+    expect(length).toEqual(5);
+    expect(type).toEqual("Uint8Array");
+  });
+});

wnfs/README.md

@@ -145,7 +145,7 @@ Namefilters are currently how we identify private node blocks in the filesystem.
 
 Check the [`examples/`][wnfs-examples] folder for more examples.
 
-[blockstore-trait]: wnfs/common/blockstore.rs#L30-L86
+[blockstore-trait]: https://github.com/wnfs-wg/rs-wnfs/blob/main/wnfs/src/common/blockstore.rs
 [hamt-wiki]: https://en.wikipedia.org/wiki/Hash_array_mapped_trie
 [ipld-spec]: https://ipld.io/
 [npm-ipld-tools]: https://www.npmjs.com/search?q=ipld

wnfs/src/common/error.rs

@@ -63,8 +63,11 @@ pub enum FsError {
     #[error("Found unexpected node type: {0:?}")]
     UnexpectedNodeType(NodeType),
 
-    #[error("Could not compute in-between ratchet {0}")]
+    #[error("Cannot compute in-between ratchet {0}")]
     NoIntermediateRatchet(PreviousErr),
+
+    #[error("Cannot find shard for file content")]
+    FileShardNotFound,
 }
 
 pub fn error<T>(err: impl std::error::Error + Send + Sync + 'static) -> Result<T> {