From 0b1499aa105afa3562aa7bb6fa591814bca1ac4c Mon Sep 17 00:00:00 2001 From: Igor Katson Date: Thu, 2 May 2024 22:08:00 +0100 Subject: [PATCH] Make all experimental storages optional --- crates/librqbit/Cargo.toml | 7 +-- crates/librqbit/examples/custom_storage.rs | 52 +++++++++++++++++-- .../librqbit/src/storage/middleware/slow.rs | 4 ++ .../librqbit/src/storage/middleware/timing.rs | 4 ++ .../storage/middleware/write_through_cache.rs | 7 +++ crates/librqbit/src/storage/mod.rs | 14 ++++- crates/librqbit/src/tests/e2e_stream.rs | 7 +-- 7 files changed, 81 insertions(+), 14 deletions(-) diff --git a/crates/librqbit/Cargo.toml b/crates/librqbit/Cargo.toml index 3f1e1a03..77052bfb 100644 --- a/crates/librqbit/Cargo.toml +++ b/crates/librqbit/Cargo.toml @@ -70,9 +70,10 @@ tokio-util = "0.7.10" bytes = "1.5.0" rlimit = "0.10.1" async-stream = "0.3.5" -memmap2 = "0.9.4" -rand_distr = "0.4.3" -lru = "0.12.3" +memmap2 = { version = "0.9.4" } + +rand_distr = { version = "0.4.3", optional = true } +lru = { version = "0.12.3", optional = true } [dev-dependencies] futures = { version = "0.3" } diff --git a/crates/librqbit/examples/custom_storage.rs b/crates/librqbit/examples/custom_storage.rs index 84754b41..3463e47b 100644 --- a/crates/librqbit/examples/custom_storage.rs +++ b/crates/librqbit/examples/custom_storage.rs @@ -1,11 +1,55 @@ -use std::time::Duration; - use librqbit::{ - storage::{examples::mmap::MmapStorageFactory, StorageFactoryExt}, + storage::{StorageFactory, StorageFactoryExt, TorrentStorage}, SessionOptions, }; use tracing::info; +use std::time::Duration; + +#[derive(Default, Clone, Copy)] +struct CustomStorageFactory { + _some_state_used_to_create_per_torrent_storage: (), +} + +#[derive(Default, Clone, Copy)] +struct CustomStorage { + _some_state_for_per_torrent_storage: (), +} + +impl StorageFactory for CustomStorageFactory { + type Storage = CustomStorage; + + fn init_storage(&self, _info: &librqbit::ManagedTorrentInfo) -> anyhow::Result { + Ok(CustomStorage::default()) + } + + fn clone_box(&self) -> librqbit::storage::BoxStorageFactory { + self.boxed() + } +} + +impl TorrentStorage for CustomStorage { + fn pread_exact(&self, _file_id: usize, _offset: u64, _buf: &mut [u8]) -> anyhow::Result<()> { + anyhow::bail!("not implemented") + } + + fn pwrite_all(&self, _file_id: usize, _offset: u64, _buf: &[u8]) -> anyhow::Result<()> { + anyhow::bail!("not implemented") + } + + fn remove_file(&self, _file_id: usize, _filename: &std::path::Path) -> anyhow::Result<()> { + anyhow::bail!("not implemented") + } + + fn ensure_file_length(&self, _file_id: usize, _length: u64) -> anyhow::Result<()> { + anyhow::bail!("not implemented") + } + + fn take(&self) -> anyhow::Result> { + anyhow::bail!("not implemented") + } +} + #[tokio::main] async fn main() -> anyhow::Result<()> { // Output logs to console. @@ -31,7 +75,7 @@ async fn main() -> anyhow::Result<()> { include_bytes!("../resources/ubuntu-21.04-live-server-amd64.iso.torrent").into(), ), Some(librqbit::AddTorrentOptions { - storage_factory: Some(MmapStorageFactory::default().boxed()), + storage_factory: Some(CustomStorageFactory::default().boxed()), paused: false, ..Default::default() }), diff --git a/crates/librqbit/src/storage/middleware/slow.rs b/crates/librqbit/src/storage/middleware/slow.rs index d588574b..496d41e2 100644 --- a/crates/librqbit/src/storage/middleware/slow.rs +++ b/crates/librqbit/src/storage/middleware/slow.rs @@ -1,3 +1,7 @@ +/* +A storage middleware that slows down the underlying storage. +*/ + use std::time::Duration; use rand_distr::Distribution; diff --git a/crates/librqbit/src/storage/middleware/timing.rs b/crates/librqbit/src/storage/middleware/timing.rs index 90123841..081a83b7 100644 --- a/crates/librqbit/src/storage/middleware/timing.rs +++ b/crates/librqbit/src/storage/middleware/timing.rs @@ -1,3 +1,7 @@ +/* +A storage middleware that logs the time underlying storage operations took. +*/ + use crate::storage::{StorageFactory, StorageFactoryExt, TorrentStorage}; #[derive(Clone)] diff --git a/crates/librqbit/src/storage/middleware/write_through_cache.rs b/crates/librqbit/src/storage/middleware/write_through_cache.rs index 7b064b20..0fa3a379 100644 --- a/crates/librqbit/src/storage/middleware/write_through_cache.rs +++ b/crates/librqbit/src/storage/middleware/write_through_cache.rs @@ -1,3 +1,10 @@ +/* +A storage middleware that caches pieces in memory, so that subsequent reads (for checksumming) are +free. + +An example, untested and unproven to be useful. +*/ + use std::num::NonZeroUsize; use anyhow::Context; diff --git a/crates/librqbit/src/storage/mod.rs b/crates/librqbit/src/storage/mod.rs index 33c89f3c..d84b500b 100644 --- a/crates/librqbit/src/storage/mod.rs +++ b/crates/librqbit/src/storage/mod.rs @@ -1,5 +1,9 @@ -pub mod examples; pub mod filesystem; + +#[cfg(feature = "storage_examples")] +pub mod examples; + +#[cfg(feature = "storage_middleware")] pub mod middleware; use std::{ @@ -65,14 +69,22 @@ impl StorageFactory for Box { } pub trait TorrentStorage: Send + Sync { + /// Given a file_id (which you can get more info from in init_storage() through torrent info) + /// read buf.len() bytes into buf at offset. fn pread_exact(&self, file_id: usize, offset: u64, buf: &mut [u8]) -> anyhow::Result<()>; + /// Given a file_id (which you can get more info from in init_storage() through torrent info) + /// write buf.len() bytes into the file at offset. fn pwrite_all(&self, file_id: usize, offset: u64, buf: &[u8]) -> anyhow::Result<()>; + /// Remove a file from the storage. If not supported, or it doesn't matter, just return Ok(()) fn remove_file(&self, file_id: usize, filename: &Path) -> anyhow::Result<()>; + /// E.g. for filesystem backend ensure that the file has a certain length, and grow/shrink as needed. fn ensure_file_length(&self, file_id: usize, length: u64) -> anyhow::Result<()>; + /// Replace the current storage with a dummy, and return a new one that should be used instead. + /// This is used to make the underlying object useless when e.g. pausing the torrent. fn take(&self) -> anyhow::Result>; } diff --git a/crates/librqbit/src/tests/e2e_stream.rs b/crates/librqbit/src/tests/e2e_stream.rs index e8c95564..97c65ead 100644 --- a/crates/librqbit/src/tests/e2e_stream.rs +++ b/crates/librqbit/src/tests/e2e_stream.rs @@ -4,11 +4,7 @@ use anyhow::Context; use tokio::{io::AsyncReadExt, time::timeout}; use tracing::info; -use crate::{ - create_torrent, - storage::{examples::inmemory::InMemoryExampleStorageFactory, StorageFactoryExt}, - AddTorrent, CreateTorrentOptions, Session, -}; +use crate::{create_torrent, AddTorrent, CreateTorrentOptions, Session}; use super::test_util::create_default_random_dir_with_torrents; @@ -86,7 +82,6 @@ async fn e2e_stream() -> anyhow::Result<()> { AddTorrent::from_bytes(torrent.as_bytes()?), Some(crate::AddTorrentOptions { paused: false, - storage_factory: Some(InMemoryExampleStorageFactory::default().boxed()), initial_peers: Some(vec![peer]), ..Default::default() }),