Refactor module storage

Currently some generic data structures and functions are located in storage::physical module. Other storage backend like MySQL also depends on these data structures or functions. So this commits moves them into an upper-level - the storage module itself, for more clear code readability and better module hierachy.
Tongsuo-Project · May 13, 2024 · 0d39f30 · 0d39f30
1 parent 7e8c441
commit 0d39f30
Show file tree

Hide file tree

Showing 14 changed files with 194 additions and 203 deletions.
diff --git a/src/cli/command/server.rs b/src/cli/command/server.rs
@@ -21,7 +21,7 @@ use crate::{
     core::Core,
     errors::RvError,
     http,
-    storage::{barrier_aes_gcm, physical},
+    storage,
     EXIT_CODE_INSUFFICIENT_PARAMS, EXIT_CODE_LOAD_CONFIG_FAILURE, EXIT_CODE_OK,
 };
 
@@ -109,9 +109,9 @@ pub fn main(config_path: &str) -> Result<(), RvError> {
 
     let server = actix_rt::System::new();
 
-    let backend = physical::new_backend(storage.stype.as_str(), &storage.config).unwrap();
+    let backend = storage::new_backend(storage.stype.as_str(), &storage.config).unwrap();
 
-    let barrier = barrier_aes_gcm::AESGCMBarrier::new(Arc::clone(&backend));
+    let barrier = storage::barrier_aes_gcm::AESGCMBarrier::new(Arc::clone(&backend));
 
     let core = Arc::new(RwLock::new(Core { physical: backend, barrier: Arc::new(barrier), ..Default::default() }));
 

diff --git a/src/core.rs b/src/core.rs
@@ -32,7 +32,8 @@ use crate::{
         barrier_aes_gcm,
         barrier_view::BarrierView,
         physical,
-        physical::{Backend as PhysicalBackend, BackendEntry as PhysicalBackendEntry},
+        Backend as PhysicalBackend,
+        BackendEntry as PhysicalBackendEntry,
     },
 };
 
@@ -435,7 +436,7 @@ mod test {
     use serde_json::Value;
 
     use super::*;
-    use crate::storage::{barrier_aes_gcm, physical};
+    use crate::storage;
 
     #[test]
     fn test_core_init() {
@@ -448,8 +449,8 @@ mod test {
         let mut conf: HashMap<String, Value> = HashMap::new();
         conf.insert("path".to_string(), Value::String(dir.to_string_lossy().into_owned()));
 
-        let backend = physical::new_backend("file", &conf).unwrap();
-        let barrier = barrier_aes_gcm::AESGCMBarrier::new(Arc::clone(&backend));
+        let backend = storage::new_backend("file", &conf).unwrap();
+        let barrier = storage::barrier_aes_gcm::AESGCMBarrier::new(Arc::clone(&backend));
         let router = Arc::new(Router::new());
         let mounts = MountTable::new();
         let core = Arc::new(RwLock::new(Core {
@@ -499,8 +500,8 @@ mod test {
         let mut conf: HashMap<String, Value> = HashMap::new();
         conf.insert("path".to_string(), Value::String(dir.to_string_lossy().into_owned()));
 
-        let backend = physical::new_backend("file", &conf).unwrap();
-        let barrier = barrier_aes_gcm::AESGCMBarrier::new(Arc::clone(&backend));
+        let backend = storage::new_backend("file", &conf).unwrap();
+        let barrier = storage::barrier_aes_gcm::AESGCMBarrier::new(Arc::clone(&backend));
 
         let core = Arc::new(RwLock::new(Core { physical: backend, barrier: Arc::new(barrier), ..Default::default() }));
 

diff --git a/src/logical/backend.rs b/src/logical/backend.rs
@@ -253,7 +253,7 @@ mod test {
     use crate::{
         logical::{Field, field::FieldTrait, FieldType, PathOperation},
         new_fields, new_fields_internal, new_path, new_path_internal, new_secret, new_secret_internal,
-        storage::{barrier_aes_gcm::AESGCMBarrier, physical},
+        storage,
     };
 
     struct MyTest;
@@ -307,9 +307,9 @@ mod test {
         let mut conf: HashMap<String, Value> = HashMap::new();
         conf.insert("path".to_string(), Value::String(dir.to_string_lossy().into_owned()));
 
-        let backend = physical::new_backend("file", &conf).unwrap();
+        let backend = storage::new_backend("file", &conf).unwrap();
 
-        let barrier = AESGCMBarrier::new(Arc::clone(&backend));
+        let barrier = storage::barrier_aes_gcm::AESGCMBarrier::new(Arc::clone(&backend));
 
         let mut logical_backend = new_logical_backend!({
             paths: [
@@ -468,9 +468,9 @@ mod test {
         let mut conf: HashMap<String, Value> = HashMap::new();
         conf.insert("path".to_string(), Value::String(dir.to_string_lossy().into_owned()));
 
-        let backend = physical::new_backend("file", &conf).unwrap();
+        let backend = storage::new_backend("file", &conf).unwrap();
 
-        let barrier = AESGCMBarrier::new(Arc::clone(&backend));
+        let barrier = storage::barrier_aes_gcm::AESGCMBarrier::new(Arc::clone(&backend));
 
         let mut logical_backend = new_logical_backend!({
             paths: [

diff --git a/src/modules/credential/userpass/mod.rs b/src/modules/credential/userpass/mod.rs
@@ -144,7 +144,7 @@ mod test {
     use crate::{
         core::{Core, SealConfig},
         logical::{Operation, Request},
-        storage::{barrier_aes_gcm, physical},
+        storage,
     };
 
     fn test_read_api(core: &Core, token: &str, path: &str, is_ok: bool) -> Result<Option<Response>, RvError> {
@@ -270,9 +270,9 @@ mod test {
         let mut conf: HashMap<String, Value> = HashMap::new();
         conf.insert("path".to_string(), Value::String(dir.to_string_lossy().into_owned()));
 
-        let backend = physical::new_backend("file", &conf).unwrap();
+        let backend = storage::new_backend("file", &conf).unwrap();
 
-        let barrier = barrier_aes_gcm::AESGCMBarrier::new(Arc::clone(&backend));
+        let barrier = storage::barrier_aes_gcm::AESGCMBarrier::new(Arc::clone(&backend));
 
         let c = Arc::new(RwLock::new(Core { physical: backend, barrier: Arc::new(barrier), ..Default::default() }));
 

diff --git a/src/modules/pki/mod.rs b/src/modules/pki/mod.rs
@@ -161,7 +161,7 @@ mod test {
     use crate::{
         core::{Core, SealConfig},
         logical::{Operation, Request},
-        storage::{barrier_aes_gcm, physical},
+        storage,
     };
 
     const CA_CERT_PEM: &str = r#"
@@ -1590,9 +1590,9 @@ xxxxxxxxxxxxxx
         let mut conf: HashMap<String, Value> = HashMap::new();
         conf.insert("path".to_string(), Value::String(dir.to_string_lossy().into_owned()));
 
-        let backend = physical::new_backend("file", &conf).unwrap();
+        let backend = storage::new_backend("file", &conf).unwrap();
 
-        let barrier = barrier_aes_gcm::AESGCMBarrier::new(Arc::clone(&backend));
+        let barrier = storage::barrier_aes_gcm::AESGCMBarrier::new(Arc::clone(&backend));
 
         let c = Arc::new(RwLock::new(Core { physical: backend, barrier: Arc::new(barrier), ..Default::default() }));
 

diff --git a/src/storage/barrier_aes_gcm.rs b/src/storage/barrier_aes_gcm.rs
@@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize};
 
 use super::{
     barrier::{SecurityBarrier, BARRIER_INIT_PATH},
-    physical::{Backend, BackendEntry},
+    Backend, BackendEntry,
     Storage, StorageEntry,
 };
 use crate::errors::RvError;
@@ -340,7 +340,7 @@ mod test {
         let mut key = vec![0u8; cipher.key_length()];
         thread_rng().fill(key.as_mut_slice());
 
-        let backend = physical::new_backend("file", &conf).unwrap();
+        let backend = new_backend("file", &conf).unwrap();
 
         let barrier = AESGCMBarrier {
             backend,
@@ -380,7 +380,7 @@ mod test {
             110, 162, 1, 9, 51, 16, 75, 157, 129,
         ];
 
-        let backend = physical::new_backend("file", &conf).unwrap();
+        let backend = new_backend("file", &conf).unwrap();
 
         let barrier = AESGCMBarrier {
             backend,
@@ -412,7 +412,7 @@ mod test {
         let mut conf: HashMap<String, Value> = HashMap::new();
         conf.insert("path".to_string(), Value::String(dir.to_string_lossy().into_owned()));
 
-        let backend = physical::new_backend("file", &conf).unwrap();
+        let backend = new_backend("file", &conf).unwrap();
 
         let barrier = AESGCMBarrier::new(Arc::clone(&backend));
 
@@ -459,7 +459,7 @@ mod test {
         let mut conf: HashMap<String, Value> = HashMap::new();
         conf.insert("path".to_string(), Value::String(dir.to_string_lossy().into_owned()));
 
-        let backend = physical::new_backend("file", &conf).unwrap();
+        let backend = new_backend("file", &conf).unwrap();
 
         let barrier = AESGCMBarrier::new(Arc::clone(&backend));
 

diff --git a/src/storage/barrier_view.rs b/src/storage/barrier_view.rs
@@ -123,7 +123,7 @@ mod test {
         let mut key = vec![0u8; 32];
         thread_rng().fill(key.as_mut_slice());
 
-        let backend = physical::new_backend("file", &conf);
+        let backend = new_backend("file", &conf);
         assert!(backend.is_ok());
         let backend = backend.unwrap();
         let aes_gcm_view = barrier_aes_gcm::AESGCMBarrier::new(Arc::clone(&backend));

diff --git a/src/storage/mod.rs b/src/storage/mod.rs
@@ -14,7 +14,9 @@
 //! Typical storage types may be direct file, databases, remote network filesystem and etc.
 //! Different strage types are all as sub-module of this module.
 
+use std::{collections::HashMap, sync::Arc};
 use serde::{Deserialize, Serialize};
+use serde_json::Value;
 
 use crate::errors::RvError;
 
@@ -54,3 +56,157 @@ impl StorageEntry {
         Ok(StorageEntry { key: k.to_string(), value: data.into_bytes() })
     }
 }
+
+pub trait Backend: Send + Sync {
+    //! This trait decsribes the generic methods that a storage backend needs to implement.
+    fn list(&self, prefix: &str) -> Result<Vec<String>, RvError>;
+    fn get(&self, key: &str) -> Result<Option<BackendEntry>, RvError>;
+    fn put(&self, entry: &BackendEntry) -> Result<(), RvError>;
+    fn delete(&self, key: &str) -> Result<(), RvError>;
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct BackendEntry {
+    pub key: String,
+    pub value: Vec<u8>,
+}
+
+/// this is a generic function that instantiates different storage backends.
+pub fn new_backend(t: &str, conf: &HashMap<String, Value>) -> Result<Arc<dyn Backend>, RvError> {
+    match t {
+        "file" => {
+            let backend = physical::file::FileBackend::new(conf)?;
+            Ok(Arc::new(backend))
+        },
+        #[cfg(feature = "storage_mysql")]
+        "mysql" => {
+            let backend = mysql::mysql_backend::MysqlBackend::new(conf)?;
+            Ok(Arc::new(backend))
+        },
+        "mock" => Ok(Arc::new(physical::mock::MockBackend::new())),
+        _ => Err(RvError::ErrPhysicalTypeInvalid),
+    }
+}
+
+#[cfg(test)]
+pub mod test {
+    use std::{collections::HashMap, env, fs};
+
+    use go_defer::defer;
+    use serde_json::Value;
+
+    use crate::storage::{new_backend, Backend, BackendEntry};
+
+    #[test]
+    fn test_new_backend() {
+        let dir = env::temp_dir().join("rusty_vault_test_new_backend");
+        assert!(fs::create_dir(&dir).is_ok());
+        defer! (
+            assert!(fs::remove_dir_all(&dir).is_ok());
+        );
+
+        let mut conf: HashMap<String, Value> = HashMap::new();
+        conf.insert("path".to_string(), Value::String(dir.to_string_lossy().into_owned()));
+
+        let backend = new_backend("file", &conf);
+        assert!(backend.is_ok());
+
+        let backend = new_backend("foo", &conf);
+        assert!(!backend.is_ok());
+    }
+
+    pub fn test_backend(backend: &dyn Backend) {
+        // Should be empty
+        let keys = backend.list("");
+        assert!(keys.is_ok());
+        assert_eq!(keys.unwrap().len(), 0);
+
+        let keys = backend.list("bar");
+        assert!(keys.is_ok());
+        assert_eq!(keys.unwrap().len(), 0);
+
+        // Delete should work if it does not exist
+        let res = backend.delete("bar");
+        assert!(res.is_ok());
+
+        // Get should work, but result is None
+        let res = backend.get("bar");
+        assert!(res.is_ok());
+        assert_eq!(res.unwrap(), None);
+
+        // Make an Entry
+        let entry = BackendEntry { key: "bar".to_string(), value: "test".as_bytes().to_vec() };
+
+        let res = backend.put(&entry);
+        assert!(res.is_ok());
+
+        // Get should ok
+        let res = backend.get("bar");
+        assert!(res.is_ok());
+        match res.unwrap() {
+            Some(e) => {
+                assert_eq!(e, entry);
+            }
+            None => panic!("Get should ok!"),
+        }
+
+        // List should not be empty
+        let keys = backend.list("");
+        assert!(keys.is_ok());
+        let keys = keys.unwrap();
+        assert_eq!(keys.len(), 1);
+        assert_eq!(keys[0], "bar".to_string());
+
+        // Delete should ok
+        let res = backend.delete("bar");
+        assert!(res.is_ok());
+
+        // List should be empty
+        let keys = backend.list("");
+        assert!(keys.is_ok());
+        let keys = keys.unwrap();
+        assert_eq!(keys.len(), 0);
+
+        // Get should work, but result is None
+        let res = backend.get("bar");
+        assert!(res.is_ok());
+        assert_eq!(res.unwrap(), None);
+    }
+
+    pub fn test_backend_list_prefix(backend: &dyn Backend) {
+        let entry1 = BackendEntry { key: "bar".to_string(), value: "test".as_bytes().to_vec() };
+        let entry2 = BackendEntry { key: "bar/foo".to_string(), value: "test".as_bytes().to_vec() };
+        let entry3 = BackendEntry { key: "bar/foo/goo".to_string(), value: "test".as_bytes().to_vec() };
+
+        let res = backend.put(&entry1);
+        assert!(res.is_ok());
+
+        let res = backend.put(&entry2);
+        assert!(res.is_ok());
+
+        let res = backend.put(&entry3);
+        assert!(res.is_ok());
+
+        // Scan the root
+        let keys = backend.list("");
+        assert!(keys.is_ok());
+        let keys = keys.unwrap();
+        assert_eq!(keys.len(), 2);
+        assert!(keys.join("") == "barbar/" || keys.join("") == "bar/bar");
+
+        // Scan bar/
+        let keys = backend.list("bar/");
+        assert!(keys.is_ok());
+        let keys = keys.unwrap();
+        assert_eq!(keys.len(), 2);
+        assert!(keys.join("") == "foofoo/" || keys.join("") == "foo/foo");
+
+        // Scan bar/foo/
+        let keys = backend.list("bar/foo/");
+        assert!(keys.is_ok());
+        let keys = keys.unwrap();
+        assert_eq!(keys.len(), 1);
+        assert_eq!(keys[0], "goo".to_string());
+    }
+}
diff --git a/src/storage/mysql/mysql_backend.rs b/src/storage/mysql/mysql_backend.rs
@@ -14,7 +14,7 @@ use crate::schema::vault::dsl::*;
 use crate::{
     errors::RvError,
     schema::vault::vault_key,
-    storage::physical::{Backend, BackendEntry},
+    storage::{Backend, BackendEntry},
 };
 
 use super::new;
@@ -130,8 +130,8 @@ mod test {
     use serde_json::Value;
     use std::collections::HashMap;
 
-    use crate::storage::physical::test::test_backend;
-    use crate::storage::physical::test::test_backend_list_prefix;
+    use crate::storage::test::test_backend;
+    use crate::storage::test::test_backend_list_prefix;
 
     use super::MysqlBackend;
 

diff --git a/src/storage/physical/file.rs b/src/storage/physical/file.rs
@@ -8,7 +8,7 @@ use std::{
 
 use serde_json::Value;
 
-use super::{Backend, BackendEntry};
+use crate::storage::{Backend, BackendEntry};
 use crate::errors::RvError;
 
 #[derive(Debug)]
@@ -140,7 +140,7 @@ mod test {
     use go_defer::defer;
 
     use super::{
-        super::test::{test_backend, test_backend_list_prefix},
+        super::super::test::{test_backend, test_backend_list_prefix},
         *,
     };