Private File Sharding

.github/workflows/checks.yaml

@@ -80,15 +80,15 @@ jobs:
       - name: Run Tests
         run: cargo test --all-features
 
-
   wasm-js-tests:
     strategy:
       fail-fast: false
       matrix:
         rust-toolchain:
           - stable
           - nightly
-    runs-on: ubuntu-latest
+
+    runs-on: macos-latest
 
     defaults:
       run:

wnfs-bench/Cargo.toml

@@ -2,27 +2,13 @@
 name = "wnfs-bench"
 version = "0.1.9"
 description = "WNFS Benchmarks"
-keywords = ["wnfs", "webnative", "ipfs", "decentralisation", "benchmarks"]
-categories = ["filesystem", "cryptography", "web-programming", "wasm"]
 edition = "2021"
-repository = "https://github.com/wnfs-wg/rs-wnfs/tree/main/bench"
-homepage = "https://fission.codes"
-authors = ["The Fission Authors"]
-
-[dependencies]
-async-std = { version = "1.11", features = ["attributes"] }
-proptest = "1.0"
-wnfs = { path = "../wnfs", optional = true }
 
 [dev-dependencies]
+async-std = { version = "1.11", features = ["attributes"] }
 criterion = { version = "0.4", features = ["async_std"] }
-
-[lib]
-path = "lib.rs"
-bench = false
-
-[features]
-default = ["wnfs/test_strategies"]
+proptest = "1.0"
+wnfs = { path = "../wnfs", features = ["test_strategies"] }
 
 [[bench]]
 name = "hamt"

wnfs-wasm/playwright.config.ts

@@ -55,6 +55,12 @@ const config: PlaywrightTestConfig = {
         ...devices["Desktop Firefox"],
       },
     },
+    {
+      name: "safari",
+      use: {
+        ...devices["Desktop Safari"],
+      },
+    },
   ],
 
   /* Folder for test artifacts such as screenshots, videos, traces, etc. */

wnfs-wasm/src/fs/private/file.rs

@@ -1,42 +1,87 @@
 //! The bindgen API for PrivateFile.
 
+use crate::{
+    fs::{
+        utils::{self, error},
+        BlockStore, ForeignBlockStore, JsResult, Namefilter, PrivateForest, Rng,
+    },
+    value,
+};
 use chrono::{DateTime, Utc};
-use js_sys::Date;
-use wasm_bindgen::prelude::wasm_bindgen;
+use js_sys::{Date, Promise, Uint8Array};
+use std::rc::Rc;
+use wasm_bindgen::{prelude::wasm_bindgen, JsValue};
+use wasm_bindgen_futures::future_to_promise;
 use wnfs::{Id, PrivateFile as WnfsPrivateFile};
 
-use crate::fs::{JsResult, Namefilter, Rng};
-
 //--------------------------------------------------------------------------------------------------
 // Type Definitions
 //--------------------------------------------------------------------------------------------------
 
 /// A file in a WNFS public file system.
 #[wasm_bindgen]
-pub struct PrivateFile(WnfsPrivateFile);
+pub struct PrivateFile(pub(crate) Rc<WnfsPrivateFile>);
 
 //--------------------------------------------------------------------------------------------------
 // Implementations
 //--------------------------------------------------------------------------------------------------
 
 #[wasm_bindgen]
 impl PrivateFile {
-    /// Creates a new private file.
-    #[wasm_bindgen(constructor)]
-    pub fn new(
+    /// Creates an empty private file.
+    pub fn empty(parent_bare_name: Namefilter, time: &Date, mut rng: Rng) -> JsResult<PrivateFile> {
+        let time = DateTime::<Utc>::from(time);
+
+        Ok(Self(Rc::new(WnfsPrivateFile::empty(
+            parent_bare_name.0,
+            time,
+            &mut rng,
+        ))))
+    }
+
+    /// Creates a file with provided content.
+    #[wasm_bindgen(js_name = "withContent")]
+    pub fn with_content(
         parent_bare_name: Namefilter,
         time: &Date,
         content: Vec<u8>,
+        hamt: PrivateForest,
+        store: BlockStore,
         mut rng: Rng,
-    ) -> JsResult<PrivateFile> {
+    ) -> JsResult<Promise> {
+        let mut store = ForeignBlockStore(store);
         let time = DateTime::<Utc>::from(time);
 
-        Ok(PrivateFile(WnfsPrivateFile::new(
-            parent_bare_name.0,
-            time,
-            content,
-            &mut rng,
-        )))
+        Ok(future_to_promise(async move {
+            let (file, hamt) = WnfsPrivateFile::with_content(
+                parent_bare_name.0,
+                time,
+                content,
+                hamt.0,
+                &mut store,
+                &mut rng,
+            )
+            .await
+            .map_err(error("Cannot create a file with provided content"))?;
+
+            Ok(utils::create_private_file_result(file, hamt)?)
+        }))
+    }
+
+    /// Gets the entire content of a file.
+    #[wasm_bindgen(js_name = "getContent")]
+    pub fn get_content(&self, hamt: PrivateForest, store: BlockStore) -> JsResult<Promise> {
+        let file = Rc::clone(&self.0);
+        let store = ForeignBlockStore(store);
+
+        Ok(future_to_promise(async move {
+            let content = file
+                .get_content(&hamt.0, &store)
+                .await
+                .map_err(error("Cannot get content of file"))?;
+
+            Ok(value!(Uint8Array::from(content.as_slice())))
+        }))
     }
 
     /// Gets a unique id for node.

wnfs-wasm/src/fs/utils.rs

@@ -4,17 +4,27 @@ use crate::{fs::JsResult, value};
 use js_sys::{Array, Error, Object, Reflect};
 use wasm_bindgen::JsValue;
 use wnfs::{
-    private::{PrivateDirectory as WnfsPrivateDirectory, PrivateForest as WnfsPrivateForest},
+    private::{
+        PrivateDirectory as WnfsPrivateDirectory, PrivateFile as WnfsPrivateFile,
+        PrivateForest as WnfsPrivateForest,
+    },
     public::PublicDirectory as WnfsPublicDirectory,
     Metadata,
 };
 
-use super::{metadata::JsMetadata, PrivateDirectory, PrivateForest, PublicDirectory};
+use super::{metadata::JsMetadata, PrivateDirectory, PrivateFile, PrivateForest, PublicDirectory};
 
 //--------------------------------------------------------------------------------------------------
 // Functions
 //--------------------------------------------------------------------------------------------------
 
+pub(crate) fn error<E>(message: &str) -> impl FnOnce(E) -> js_sys::Error + '_
+where
+    E: Debug,
+{
+    move |e| Error::new(&format!("{message}: {e:?}"))
+}
+
 pub(crate) fn map_to_rust_vec<T, F: FnMut(JsValue) -> JsResult<T>>(
     array: &Array,
     f: F,
@@ -67,11 +77,20 @@ pub(crate) fn create_private_op_result<T: Into<JsValue>>(
     Ok(value!(op_result))
 }
 
-pub(crate) fn error<E>(message: &str) -> impl FnOnce(E) -> js_sys::Error + '_
-where
-    E: Debug,
-{
-    move |e| Error::new(&format!("{message}: {e:?}"))
+pub(crate) fn create_private_file_result(
+    file: WnfsPrivateFile,
+    hamt: Rc<WnfsPrivateForest>,
+) -> JsResult<JsValue> {
+    let op_result = Object::new();
+
+    Reflect::set(
+        &op_result,
+        &value!("file"),
+        &PrivateFile(Rc::new(file)).into(),
+    )?;
+    Reflect::set(&op_result, &value!("hamt"), &PrivateForest(hamt).into())?;
+
+    Ok(value!(op_result))
 }
 
 pub(crate) fn create_ls_entry(name: &String, metadata: &Metadata) -> JsResult<JsValue> {

wnfs-wasm/tests/private.spec.ts

@@ -339,3 +339,74 @@ test.describe("PrivateDirectory", () => {
     expect(picturesContent[0].name).toEqual("cats");
   });
 });
+
+test.describe("PrivateFile", () => {
+  test("empty can create empty file", async ({ page }) => {
+    const result = await page.evaluate(async () => {
+      const {
+        wnfs: { PrivateFile, Namefilter },
+        mock: { Rng },
+      } = await window.setup();
+
+      const rng = new Rng();
+      const file = PrivateFile.empty(new Namefilter(), new Date(), rng);
+
+      return file.getId();
+    });
+
+    expect(result).toBeDefined();
+  });
+
+  test("withContent can create file with content", async ({ page }) => {
+    const result = await page.evaluate(async () => {
+      const {
+        wnfs: { PrivateFile, PrivateForest, Namefilter },
+        mock: { MemoryBlockStore, Rng },
+      } = await window.setup();
+
+      const hamt = new PrivateForest();
+      const rng = new Rng();
+      const store = new MemoryBlockStore();
+      const { file } = await PrivateFile.withContent(
+        new Namefilter(),
+        new Date(),
+        new Uint8Array([1, 2, 3, 4, 5]),
+        hamt,
+        store,
+        rng
+      );
+
+      return file.getId();
+    });
+
+    expect(result).toBeDefined();
+  });
+
+  test("getContent can fetch file's entire content", async ({ page }) => {
+    const [length, type] = await page.evaluate(async () => {
+      const {
+        wnfs: { PrivateFile, PrivateForest, Namefilter },
+        mock: { MemoryBlockStore, Rng },
+      } = await window.setup();
+
+      const initialHamt = new PrivateForest();
+      const rng = new Rng();
+      const store = new MemoryBlockStore();
+      var { file, hamt } = await PrivateFile.withContent(
+        new Namefilter(),
+        new Date(),
+        new Uint8Array([1, 2, 3, 4, 5]),
+        initialHamt,
+        store,
+        rng
+      );
+
+      let content = await file.getContent(hamt, store);
+
+      return [content.length, content.constructor.name, content];
+    });
+
+    expect(length).toEqual(5);
+    expect(type).toEqual("Uint8Array");
+  });
+});

wnfs/src/common/error.rs

@@ -63,8 +63,11 @@ pub enum FsError {
     #[error("Found unexpected node type: {0:?}")]
     UnexpectedNodeType(NodeType),
 
-    #[error("Could not compute in-between ratchet {0}")]
+    #[error("Cannot compute in-between ratchet {0}")]
     NoIntermediateRatchet(PreviousErr),
+
+    #[error("Cannot find shard for file content")]
+    FileShardNotFound,
 }
 
 pub fn error<T>(err: impl std::error::Error + Send + Sync + 'static) -> Result<T> {

wnfs/src/common/metadata.rs

@@ -73,8 +73,8 @@ impl Metadata {
     ///
     /// metadata.upsert_mtime(time);
     ///
-    /// let imprecise_time = Utc.timestamp(time.timestamp(), 0);
-    /// assert_eq!(metadata.get_modified(), Some(imprecise_time));
+    /// let imprecise_time = Utc.timestamp_opt(time.timestamp(), 0).single();
+    /// assert_eq!(metadata.get_modified(), imprecise_time);
     /// ```
     pub fn upsert_mtime(&mut self, time: DateTime<Utc>) {
         self.0.insert("modified".into(), time.timestamp().into());
@@ -91,12 +91,12 @@ impl Metadata {
     /// let time = Utc::now();
     /// let metadata = Metadata::new(time);
     ///
-    /// let imprecise_time = Utc.timestamp(time.timestamp(), 0);
-    /// assert_eq!(metadata.get_created(), Some(imprecise_time));
+    /// let imprecise_time = Utc.timestamp_opt(time.timestamp(), 0).single();
+    /// assert_eq!(metadata.get_created(), imprecise_time);
     /// ```
     pub fn get_created(&self) -> Option<DateTime<Utc>> {
         self.0.get("created").and_then(|ipld| match ipld {
-            Ipld::Integer(i) => Some(Utc.timestamp(i64::try_from(*i).ok()?, 0)),
+            Ipld::Integer(i) => Utc.timestamp_opt(i64::try_from(*i).ok()?, 0).single(),
             _ => None,
         })
     }
@@ -112,12 +112,12 @@ impl Metadata {
     /// let time = Utc::now();
     /// let metadata = Metadata::new(time);
     ///
-    /// let imprecise_time = Utc.timestamp(time.timestamp(), 0);
-    /// assert_eq!(metadata.get_modified(), Some(imprecise_time));
+    /// let imprecise_time = Utc.timestamp_opt(time.timestamp(), 0).single();
+    /// assert_eq!(metadata.get_modified(), imprecise_time);
     /// ```
     pub fn get_modified(&self) -> Option<DateTime<Utc>> {
         self.0.get("modified").and_then(|ipld| match ipld {
-            Ipld::Integer(i) => Some(Utc.timestamp(i64::try_from(*i).ok()?, 0)),
+            Ipld::Integer(i) => Utc.timestamp_opt(i64::try_from(*i).ok()?, 0).single(),
             _ => None,
         })
     }

wnfs/src/common/mod.rs

@@ -16,8 +16,14 @@ pub use metadata::*;
 pub use pathnodes::*;
 
 //--------------------------------------------------------------------------------------------------
-// Type Definitions
+// Constants
 //--------------------------------------------------------------------------------------------------
 
 pub const HASH_BYTE_SIZE: usize = 32;
+pub const MAX_BLOCK_SIZE: usize = usize::pow(2, 18);
+
+//--------------------------------------------------------------------------------------------------
+// Type Definitions
+//--------------------------------------------------------------------------------------------------
+
 pub type HashOutput = [u8; HASH_BYTE_SIZE];