Remove the dat folders to save littlefs space

CHANGELOG.md

@@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Breaking Changes
 
+- Remove the `dat` intermediary directory in file storage ([#39][])
 - Add `delete_app_keys` and `delete_auth_keys` syscalls. ([#33][])
 
   - `delete_all_pins` now doesn't affect application keys
@@ -36,6 +37,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [#35]: https://github.com/trussed-dev/trussed-auth/pull/35
 [#36]: https://github.com/trussed-dev/trussed-auth/pull/36
 [#37]: https://github.com/trussed-dev/trussed-auth/pull/37
+[#39]: https://github.com/trussed-dev/trussed-auth/pull/39
 
 ## [0.2.2][] - 2023-04-26
 

Cargo.toml

@@ -20,12 +20,18 @@ serde-byte-array = "0.1.2"
 sha2 = { version = "0.10.6", default-features = false }
 subtle = { version = "2.4.1", default-features = false }
 trussed = { version = "0.1.0", features = ["serde-extensions"] }
+littlefs2 = "0.4.0"
 
 [dev-dependencies]
 quickcheck = { version = "1.0.3", default-features = false }
 rand_core = { version = "0.6.4", default-features = false, features = ["getrandom"] }
 trussed = { version = "0.1.0", features = ["serde-extensions", "virt"] }
+admin-app = { version = "0.1.0", features = ["migration-tests"] }
 
 [patch.crates-io]
-littlefs2 = { git =  "https://github.com/Nitrokey/littlefs2", tag = "v0.3.2-nitrokey-2" }
-trussed = { git = "https://github.com/trussed-dev/trussed.git", rev = "df720980888e3e0d5487250bd14a28db02a5f13b" }
+littlefs2 = { git = "https://github.com/sosthene-nitrokey/littlefs2.git", rev = "2b45a7559ff44260c6dd693e4cb61f54ae5efc53" }
+trussed = { git = "https://github.com/Nitrokey/trussed.git", rev = "be04182e2c74e73599a394e814d353bc4bf79484" }
+trussed-manage = { git = "https://github.com/trussed-dev/trussed-staging.git", tag = "manage-v0.1.0" }
+apdu-dispatch = { git = "https://github.com/trussed-dev/apdu-dispatch.git", rev = "915fc237103fcecc29d0f0b73391f19abf6576de" }
+ctaphid-dispatch = { git = "https://github.com/trussed-dev/ctaphid-dispatch.git", rev = "57cb3317878a8593847595319aa03ef17c29ec5b" }
+admin-app = { git = "https://github.com/Nitrokey/admin-app.git", rev = "b2df7c068f889337eb57ce8646ded9418b5ec773" }

src/backend.rs

@@ -52,6 +52,15 @@ impl fmt::Debug for HardwareKey {
     }
 }
 
+/// Filesystem layout used
+#[derive(Debug, Clone)]
+pub enum FilesystemLayout {
+    /// The default layout
+    V0,
+    /// The optimized layout, requireing the [`migrate::migrate_remove_dat`]() migration
+    V1,
+}
+
 /// A basic implementation of the [`AuthExtension`][].
 ///
 /// This implementation stores PINs together with their retry counters on the filesystem.  PINs are
@@ -75,24 +84,31 @@ impl fmt::Debug for HardwareKey {
 pub struct AuthBackend {
     location: Location,
     hw_key: HardwareKey,
+    layout: FilesystemLayout,
 }
 
 impl AuthBackend {
     /// Creates a new `AuthBackend` using the given storage location for the PINs.
-    pub fn new(location: Location) -> Self {
+    pub fn new(location: Location, layout: FilesystemLayout) -> Self {
         Self {
             location,
             hw_key: HardwareKey::None,
+            layout,
         }
     }
 
     /// Creates a new `AuthBackend` with a given key.
     ///
     /// This key is used to strengthen key generation from the pins
-    pub fn with_hw_key(location: Location, hw_key: Bytes<MAX_HW_KEY_LEN>) -> Self {
+    pub fn with_hw_key(
+        location: Location,
+        hw_key: Bytes<MAX_HW_KEY_LEN>,
+        layout: FilesystemLayout,
+    ) -> Self {
         Self {
             location,
             hw_key: HardwareKey::Raw(hw_key),
+            layout,
         }
     }
 
@@ -101,10 +117,11 @@ impl AuthBackend {
     /// Contrary to [`new`](Self::new) which uses a default `&[]` key, this will make operations depending on the hardware key to fail:
     /// - [`set_pin`](crate::AuthClient::set_pin) with `derive_key = true`
     /// - All operations on a pin that was created with `derive_key = true`
-    pub fn with_missing_hw_key(location: Location) -> Self {
+    pub fn with_missing_hw_key(location: Location, layout: FilesystemLayout) -> Self {
         Self {
             location,
             hw_key: HardwareKey::Missing,
+            layout,
         }
     }
 
@@ -215,8 +232,22 @@ impl ExtensionImpl<AuthExtension> for AuthBackend {
         // FIXME: Have a real implementation from trussed
         let mut backend_path = core_ctx.path.clone();
         backend_path.push(&PathBuf::from(BACKEND_DIR));
-        let fs = &mut resources.filestore(backend_path);
-        let global_fs = &mut resources.filestore(PathBuf::from(BACKEND_DIR));
+        let mut fs;
+        let mut global_fs;
+        match self.layout {
+            FilesystemLayout::V0 => {
+                fs = resources.filestore(backend_path);
+                global_fs = resources.filestore(PathBuf::from(BACKEND_DIR));
+            }
+            FilesystemLayout::V1 => {
+                fs = resources.raw_filestore(backend_path);
+                global_fs = resources.raw_filestore(PathBuf::from(BACKEND_DIR));
+            }
+        }
+
+        let fs = &mut fs;
+        let global_fs = &mut global_fs;
+
         let rng = &mut resources.rng()?;
         let client_id = core_ctx.path.clone();
         let keystore = &mut resources.keystore(core_ctx.path.clone())?;

src/lib.rs

@@ -63,6 +63,8 @@
 mod backend;
 mod extension;
 
+pub mod migrate;
+
 use core::str::FromStr;
 
 use serde::{Deserialize, Serialize};
@@ -71,7 +73,7 @@ use trussed::{
     types::{Bytes, PathBuf},
 };
 
-pub use backend::{AuthBackend, AuthContext, MAX_HW_KEY_LEN};
+pub use backend::{AuthBackend, AuthContext, FilesystemLayout, MAX_HW_KEY_LEN};
 pub use extension::{
     reply, request, AuthClient, AuthExtension, AuthReply, AuthRequest, AuthResult,
 };

src/migrate.rs

@@ -0,0 +1,207 @@
+// Copyright (C) Nitrokey GmbH
+// SPDX-License-Identifier: Apache-2.0 or MIT
+
+//! Helper to migrate from trussed-auth's legacy data layout to the new layout
+//!
+//! See [migrate]()
+
+use core::iter::once;
+
+use littlefs2::{io::Error, object_safe::DynFilesystem, path, path::Path};
+
+fn migrate_single(fs: &dyn DynFilesystem, path: &Path) -> Result<(), Error> {
+    let path = path.join(path!("backend-auth"));
+    let path_dat = path.join(path!("dat"));
+    let dir_res = fs.read_dir_and_then(&path_dat, &mut |dir| {
+        for f in dir.skip(2) {
+            let f = f?;
+            let new_path = path.join(f.file_name());
+            fs.rename(f.path(), &new_path)?;
+        }
+        Ok(())
+    });
+    match dir_res {
+        Ok(()) => fs.remove_dir(&path_dat),
+        Err(Error::NoSuchEntry) => Ok(()),
+        Err(_) => dir_res,
+    }
+}
+
+///  Migrate the filesystem to remove the `dat` directories
+///
+/// `apps` must be an array of paths to the apps that make use of trussed-auth
+///
+/// Migrate does not itself keep track of whether the migration was performed
+///
+/// ```rust
+///# use littlefs2::{fs::Filesystem, const_ram_storage, path};
+///# use trussed::types::{LfsResult, LfsStorage};
+///# use trussed_auth::migrate::migrate_remove_dat;
+///# const_ram_storage!(Storage, 4096);
+///# let mut storage = Storage::new();
+///# Filesystem::format(&mut storage);
+///# Filesystem::mount_and_then(&mut storage, |fs| {
+/// migrate_remove_dat(fs, &[path!("secrets"), path!("opcard")])?;
+///#     Ok(())
+///# }).unwrap();
+/// ```
+pub fn migrate_remove_dat(fs: &dyn DynFilesystem, apps: &[&Path]) -> Result<(), Error> {
+    for p in once(&path!("/")).chain(apps) {
+        migrate_single(fs, p)?;
+    }
+    Ok(())
+}
+
+#[allow(clippy::unwrap_used)]
+#[cfg(test)]
+mod tests {
+    use admin_app::migrations::test_utils::{test_migration_one, FsValues};
+
+    use super::*;
+
+    const FIDO_DAT_DIR: FsValues = FsValues::Dir(&[
+        (path!("persistent-state.cbor"), FsValues::File(137)),
+        (
+            path!("rk"),
+            FsValues::Dir(&[(
+                path!("74a6ea9213c99c2f"),
+                FsValues::Dir(&[
+                    (path!("038dfc6165b78be9"), FsValues::File(128)),
+                    (path!("1ecbbfbed8992287"), FsValues::File(122)),
+                    (path!("7c24db95312eac56"), FsValues::File(122)),
+                    (path!("978cba44dfe39871"), FsValues::File(155)),
+                    (path!("ac889a0433749726"), FsValues::File(138)),
+                ]),
+            )]),
+        ),
+    ]);
+
+    const FIDO_SEC_DIR: FsValues = FsValues::Dir(&[
+        (
+            path!("069386c3c735689061ac51b8bca9f160"),
+            FsValues::File(48),
+        ),
+        (
+            path!("233d86bfc2f196ff7c108cf23a282bd5"),
+            FsValues::File(36),
+        ),
+        (
+            path!("2bdef14a0e18d28191162f8c1599d598"),
+            FsValues::File(36),
+        ),
+        (
+            path!("3efe6394c20aa8128e27b376e226a58b"),
+            FsValues::File(36),
+        ),
+        (
+            path!("4711aa79b4834ef8e551f80e523ba8d2"),
+            FsValues::File(36),
+        ),
+        (
+            path!("b43bf8b7897087b7195b8ac53dcb5f11"),
+            FsValues::File(36),
+        ),
+    ]);
+
+    #[test]
+    fn migration_nothing() {
+        const TEST_VALUES: FsValues = FsValues::Dir(&[
+            (
+                path!("fido"),
+                FsValues::Dir(&[(path!("dat"), FIDO_DAT_DIR), (path!("sec"), FIDO_SEC_DIR)]),
+            ),
+            (
+                path!("trussed"),
+                FsValues::Dir(&[(
+                    path!("dat"),
+                    FsValues::Dir(&[(path!("rng-state.bin"), FsValues::File(32))]),
+                )]),
+            ),
+        ]);
+        test_migration_one(&TEST_VALUES, &TEST_VALUES, |fs| {
+            migrate_remove_dat(fs, &[path!("secrets"), path!("opcard")])
+        });
+    }
+
+    #[test]
+    fn migration_full() {
+        const AUTH_SECRETS_DIR: FsValues = FsValues::Dir(&[
+            (path!("application_salt"), FsValues::File(16)),
+            (path!("pin.00"), FsValues::File(118)),
+        ]);
+
+        const TEST_BEFORE: FsValues = FsValues::Dir(&[
+            (
+                path!("backend-auth"),
+                FsValues::Dir(&[(
+                    path!("dat"),
+                    FsValues::Dir(&[(path!("salt"), FsValues::File(16))]),
+                )]),
+            ),
+            (
+                path!("fido"),
+                FsValues::Dir(&[(path!("dat"), FIDO_DAT_DIR), (path!("sec"), FIDO_SEC_DIR)]),
+            ),
+            (
+                path!("secrets"),
+                FsValues::Dir(&[(
+                    path!("backend-auth"),
+                    FsValues::Dir(&[(path!("dat"), AUTH_SECRETS_DIR)]),
+                )]),
+            ),
+            (
+                path!("trussed"),
+                FsValues::Dir(&[(
+                    path!("dat"),
+                    FsValues::Dir(&[(path!("rng-state.bin"), FsValues::File(32))]),
+                )]),
+            ),
+        ]);
+
+        const TEST_AFTER: FsValues = FsValues::Dir(&[
+            (
+                path!("backend-auth"),
+                FsValues::Dir(&[(path!("salt"), FsValues::File(16))]),
+            ),
+            (
+                path!("fido"),
+                FsValues::Dir(&[(path!("dat"), FIDO_DAT_DIR), (path!("sec"), FIDO_SEC_DIR)]),
+            ),
+            (
+                path!("secrets"),
+                FsValues::Dir(&[(path!("backend-auth"), AUTH_SECRETS_DIR)]),
+            ),
+            (
+                path!("trussed"),
+                FsValues::Dir(&[(
+                    path!("dat"),
+                    FsValues::Dir(&[(path!("rng-state.bin"), FsValues::File(32))]),
+                )]),
+            ),
+        ]);
+
+        test_migration_one(&TEST_BEFORE, &TEST_AFTER, |fs| {
+            migrate_remove_dat(fs, &[path!("secrets"), path!("opcard")])
+        });
+    }
+
+    #[test]
+    fn migration_empty() {
+        const TEST_VALUES: FsValues = FsValues::Dir(&[
+            (
+                path!("fido"),
+                FsValues::Dir(&[(path!("dat"), FIDO_DAT_DIR), (path!("sec"), FIDO_SEC_DIR)]),
+            ),
+            (
+                path!("trussed"),
+                FsValues::Dir(&[(
+                    path!("dat"),
+                    FsValues::Dir(&[(path!("rng-state.bin"), FsValues::File(32))]),
+                )]),
+            ),
+        ]);
+        test_migration_one(&TEST_VALUES, &TEST_VALUES, |fs| {
+            migrate_remove_dat(fs, &[path!("secrets"), path!("opcard")])
+        });
+    }
+}

tests/backend.rs

@@ -55,18 +55,25 @@ mod dispatch {
     impl Dispatch {
         pub fn new() -> Self {
             Self {
-                auth: AuthBackend::new(Location::Internal),
+                auth: AuthBackend::new(Location::Internal, trussed_auth::FilesystemLayout::V0),
             }
         }
 
         pub fn with_hw_key(hw_key: Bytes<MAX_HW_KEY_LEN>) -> Self {
             Self {
-                auth: AuthBackend::with_hw_key(Location::Internal, hw_key),
+                auth: AuthBackend::with_hw_key(
+                    Location::Internal,
+                    hw_key,
+                    trussed_auth::FilesystemLayout::V0,
+                ),
             }
         }
         pub fn with_missing_hw_key() -> Self {
             Self {
-                auth: AuthBackend::with_missing_hw_key(Location::Internal),
+                auth: AuthBackend::with_missing_hw_key(
+                    Location::Internal,
+                    trussed_auth::FilesystemLayout::V0,
+                ),
             }
         }
     }