Skip to content
This repository was archived by the owner on Sep 21, 2024. It is now read-only.

Commit a0176f2

Browse files
committed
feat: Introduce storage migrations via ImportStorage/ExportStorage traits, and other trait-based operations.
1 parent 36cf25e commit a0176f2

File tree

16 files changed

+835
-153
lines changed

16 files changed

+835
-153
lines changed

.vscode/settings.json

+2-3
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
},
1313
"rust-analyzer.cargo.features": [
1414
"test-kubo",
15-
"helpers",
16-
"performance"
15+
"helpers"
1716
]
18-
}
17+
}

rust/noosphere-storage/Cargo.toml

+9-11
Original file line numberDiff line numberDiff line change
@@ -21,30 +21,28 @@ readme = "README.md"
2121
anyhow = { workspace = true }
2222
async-trait = "~0.1"
2323
async-stream = { workspace = true }
24-
tokio-stream = { workspace = true }
24+
base64 = "=0.21.2"
2525
cid = { workspace = true }
26-
noosphere-common = { version = "0.1.0", path = "../noosphere-common" }
27-
tracing = "~0.1"
28-
ucan = { workspace = true }
26+
instant = { version = "0.1.12", features = ["wasm-bindgen"] }
2927
libipld-core = { workspace = true }
3028
libipld-cbor = { workspace = true }
29+
noosphere-common = { version = "0.1.0", path = "../noosphere-common" }
30+
rand = { workspace = true }
3131
serde = { workspace = true }
32-
base64 = "=0.21.2"
32+
tokio-stream = { workspace = true }
33+
tracing = "~0.1"
34+
ucan = { workspace = true }
3335
url = { version = "^2" }
36+
witty-phrase-generator = "~0.2"
3437

3538
[dev-dependencies]
36-
witty-phrase-generator = "~0.2"
3739
wasm-bindgen-test = { workspace = true }
38-
rand = { workspace = true }
3940
noosphere-core-dev = { path = "../noosphere-core", features = ["helpers"], package = "noosphere-core" }
4041
noosphere-common = { path = "../noosphere-common", features = ["helpers"] }
41-
instant = { version = "0.1.12", features = ["wasm-bindgen"] }
42-
43-
[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
44-
tempfile = { workspace = true }
4542

4643
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
4744
sled = "~0.34"
45+
tempfile = { workspace = true }
4846
tokio = { workspace = true, features = ["full"] }
4947
rocksdb = { version = "0.21.0", optional = true }
5048

rust/noosphere-storage/examples/bench/main.rs

+2-4
Original file line numberDiff line numberDiff line change
@@ -132,17 +132,15 @@ impl BenchmarkStorage {
132132
))]
133133
let (storage, storage_name) = {
134134
(
135-
noosphere_storage::SledStorage::new(noosphere_storage::SledStorageInit::Path(
136-
storage_path.into(),
137-
))?,
135+
noosphere_storage::SledStorage::new(&storage_path)?,
138136
"SledDbStorage",
139137
)
140138
};
141139

142140
#[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))]
143141
let (storage, storage_name) = {
144142
(
145-
noosphere_storage::RocksDbStorage::new(storage_path.into())?,
143+
noosphere_storage::RocksDbStorage::new(&storage_path)?,
146144
"RocksDbStorage",
147145
)
148146
};

rust/noosphere-storage/src/backup.rs

+171
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
use crate::storage::Storage;
2+
use anyhow::Result;
3+
use async_trait::async_trait;
4+
use noosphere_common::ConditionalSend;
5+
use std::path::{Path, PathBuf};
6+
7+
#[cfg(not(target_arch = "wasm32"))]
8+
fn create_backup_path<P: AsRef<Path>>(path: P) -> Result<PathBuf> {
9+
use instant::SystemTime;
10+
use rand::Rng;
11+
12+
let mut path = path.as_ref().to_owned();
13+
let timestamp = SystemTime::UNIX_EPOCH
14+
.elapsed()
15+
.map_err(|_| anyhow::anyhow!("Could not generate timestamp."))?
16+
.as_secs();
17+
let nonce = rand::thread_rng().gen::<u32>();
18+
path.set_extension(format!("backup.{}-{}", timestamp, nonce));
19+
Ok(path)
20+
}
21+
/*
22+
impl TryFrom<PathBuf> for BackupPath {
23+
fn try_from(value: PathBuf) -> Result<Self> {
24+
let file_name = value
25+
.file_name()
26+
.ok_or_else(|| anyhow::anyhow!("Could not derive file name."))?
27+
.to_str()
28+
.ok_or_else(|| anyhow::anyhow!("Could not decode file name."))?;
29+
match file_name.split('.').collect::<Vec<_>>()[..] {
30+
[source, "backup", time_and_nonce] => match time_and_nonce.split('-').collect()[..] {
31+
[time, nonce] => Duration::from_secs(),
32+
},
33+
_ => Err(anyhow::anyhow!("Invalid backup path format.")),
34+
}
35+
}
36+
}
37+
*/
38+
39+
/// [Storage] that can be backed up and restored.
40+
/// [FsBackedStorage] types get a blanket implementation.
41+
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
42+
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
43+
pub trait BackupStorage: Storage {
44+
/// Backup [Storage] located at `path`, moving to a backup location.
45+
async fn backup<P: AsRef<Path> + ConditionalSend>(path: P) -> Result<PathBuf>;
46+
/// Backup [Storage] at `restore_to`, moving [Storage] from `backup_path` to `restore_to`.
47+
async fn restore<P: AsRef<Path> + ConditionalSend, Q: AsRef<Path> + ConditionalSend>(
48+
backup_path: P,
49+
restore_to: Q,
50+
) -> Result<PathBuf>;
51+
/// List paths to backups for `path`.
52+
async fn list_backups<P: AsRef<Path> + ConditionalSend>(path: P) -> Result<Vec<PathBuf>>;
53+
}
54+
55+
#[cfg(not(target_arch = "wasm32"))]
56+
#[async_trait]
57+
impl<T> BackupStorage for T
58+
where
59+
T: crate::FsBackedStorage,
60+
{
61+
async fn backup<P: AsRef<Path> + ConditionalSend>(path: P) -> Result<PathBuf> {
62+
let backup_path = create_backup_path(path.as_ref())?;
63+
T::rename(path, &backup_path).await?;
64+
Ok(backup_path)
65+
}
66+
67+
async fn restore<P: AsRef<Path> + ConditionalSend, Q: AsRef<Path> + ConditionalSend>(
68+
backup_path: P,
69+
restore_to: Q,
70+
) -> Result<PathBuf> {
71+
let restoration_path = restore_to.as_ref().to_owned();
72+
let original_backup = T::backup(&restoration_path).await?;
73+
T::rename(backup_path, &restoration_path).await?;
74+
Ok(original_backup)
75+
}
76+
77+
async fn list_backups<P: AsRef<Path> + ConditionalSend>(path: P) -> Result<Vec<PathBuf>> {
78+
let mut backups = vec![];
79+
let matcher = format!(
80+
"{}.backup.",
81+
path.as_ref()
82+
.file_name()
83+
.ok_or_else(|| anyhow::anyhow!("Could not stringify path."))?
84+
.to_str()
85+
.ok_or_else(|| anyhow::anyhow!("Could not stringify path."))?
86+
);
87+
let parent_dir = path
88+
.as_ref()
89+
.parent()
90+
.ok_or_else(|| anyhow::anyhow!("Could not find storage parent directory."))?;
91+
let mut stream = tokio::fs::read_dir(parent_dir).await?;
92+
while let Ok(Some(entry)) = stream.next_entry().await {
93+
if let Ok(file_name) = entry.file_name().into_string() {
94+
if file_name.starts_with(&matcher) {
95+
backups.push(entry.path());
96+
}
97+
}
98+
}
99+
Ok(backups)
100+
}
101+
}
102+
103+
#[cfg(all(not(target_arch = "wasm32"), test))]
104+
mod test {
105+
use crate::{OpenStorage, PreferredPlatformStorage, Store};
106+
107+
use super::*;
108+
109+
#[tokio::test]
110+
pub async fn it_can_backup_storages() -> Result<()> {
111+
noosphere_core_dev::tracing::initialize_tracing(None);
112+
113+
let temp_dir = tempfile::TempDir::new()?;
114+
let db_source = temp_dir.path().join("db");
115+
116+
{
117+
let storage = PreferredPlatformStorage::open(&db_source).await?;
118+
let mut store = storage.get_key_value_store("links").await?;
119+
store.write(b"1", b"1").await?;
120+
}
121+
122+
let backup_1 = PreferredPlatformStorage::backup(&db_source).await?;
123+
124+
{
125+
let storage = PreferredPlatformStorage::open(&db_source).await?;
126+
let mut store = storage.get_key_value_store("links").await?;
127+
assert!(store.read(b"1").await?.is_none(), "Backup is a move");
128+
store.write(b"2", b"2").await?;
129+
}
130+
131+
let backup_2 = PreferredPlatformStorage::backup(&db_source).await?;
132+
133+
{
134+
let storage = PreferredPlatformStorage::open(&db_source).await?;
135+
let mut store = storage.get_key_value_store("links").await?;
136+
assert!(store.read(b"1").await?.is_none(), "Backup is a move");
137+
assert!(store.read(b"2").await?.is_none(), "Backup is a move");
138+
store.write(b"3", b"3").await?;
139+
}
140+
141+
let backups = PreferredPlatformStorage::list_backups(&db_source).await?;
142+
assert_eq!(backups.len(), 2);
143+
assert!(backups.contains(&backup_1));
144+
assert!(backups.contains(&backup_2));
145+
146+
let backup_3 = PreferredPlatformStorage::restore(&backup_1, &db_source).await?;
147+
{
148+
let storage = PreferredPlatformStorage::open(&db_source).await?;
149+
let store = storage.get_key_value_store("links").await?;
150+
assert_eq!(store.read(b"1").await?.unwrap(), b"1");
151+
assert!(store.read(b"2").await?.is_none(), "Backup is a move");
152+
assert!(store.read(b"3").await?.is_none(), "Backup is a move");
153+
}
154+
155+
let backups = PreferredPlatformStorage::list_backups(db_source).await?;
156+
assert_eq!(backups.len(), 2);
157+
assert!(
158+
backups.contains(&backup_3),
159+
"contains backup from restoration."
160+
);
161+
assert!(
162+
!backups.contains(&backup_1),
163+
"moves backup that was restored."
164+
);
165+
assert!(
166+
backups.contains(&backup_2),
167+
"contains backups that were untouched."
168+
);
169+
Ok(())
170+
}
171+
}

rust/noosphere-storage/src/fs.rs

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
use crate::storage::Storage;
2+
use anyhow::Result;
3+
use async_trait::async_trait;
4+
use noosphere_common::ConditionalSend;
5+
use std::path::Path;
6+
7+
/// [Storage] that is based on a file system. Implementing [FsBackedStorage]
8+
/// provides blanket implementations for other trait-based [Storage] operations.
9+
#[cfg(not(target_arch = "wasm32"))]
10+
#[async_trait]
11+
pub trait FsBackedStorage: Storage + Sized {
12+
/// Deletes the storage located at `path` directory. Returns `Ok(())` if
13+
/// the directory is successfully removed, or if it already does not exist.
14+
async fn delete<P: AsRef<Path> + ConditionalSend>(path: P) -> Result<()> {
15+
match std::fs::metadata(path.as_ref()) {
16+
Ok(_) => std::fs::remove_dir_all(path.as_ref()).map_err(|e| e.into()),
17+
Err(_) => Ok(()),
18+
}
19+
}
20+
21+
/// Moves the storage located at `from` to the `to` location.
22+
async fn rename<P: AsRef<Path> + ConditionalSend, Q: AsRef<Path> + ConditionalSend>(
23+
from: P,
24+
to: Q,
25+
) -> Result<()> {
26+
std::fs::rename(from, to).map_err(|e| e.into())
27+
}
28+
}
29+
30+
/// [Storage] that is based on a file system.
31+
#[cfg(target_arch = "wasm32")]
32+
#[async_trait(?Send)]
33+
pub trait FsBackedStorage: Storage + Sized {
34+
/// Deletes the storage located at `path` directory. Returns `Ok(())` if
35+
/// the directory is successfully removed, or if it already does not exist.
36+
async fn delete<P: AsRef<Path> + ConditionalSend>(path: P) -> Result<()>;
37+
38+
/// Moves the storage located at `from` to the `to` location.
39+
async fn rename<P: AsRef<Path> + ConditionalSend, Q: AsRef<Path> + ConditionalSend>(
40+
from: P,
41+
to: Q,
42+
) -> Result<()>;
43+
}
44+
45+
#[cfg(not(target_arch = "wasm32"))]
46+
#[async_trait]
47+
impl<T> crate::ops::DeleteStorage for T
48+
where
49+
T: FsBackedStorage,
50+
{
51+
async fn delete<P: AsRef<Path> + ConditionalSend>(path: P) -> Result<()> {
52+
<T as FsBackedStorage>::delete(path).await
53+
}
54+
}
55+
56+
#[cfg(not(target_arch = "wasm32"))]
57+
#[async_trait]
58+
impl<T> crate::ops::RenameStorage for T
59+
where
60+
T: FsBackedStorage,
61+
{
62+
async fn rename<P: AsRef<Path> + ConditionalSend, Q: AsRef<Path> + ConditionalSend>(
63+
from: P,
64+
to: Q,
65+
) -> Result<()> {
66+
<T as FsBackedStorage>::rename(from, to).await
67+
}
68+
}

rust/noosphere-storage/src/helpers.rs

-34
This file was deleted.

0 commit comments

Comments
 (0)