From e38ad6ad7f81b16bc20bcfbbc77e01976e3ec301 Mon Sep 17 00:00:00 2001 From: Croxx Date: Mon, 29 May 2023 15:04:35 +0800 Subject: [PATCH] chore: wrapper cache type, reorg utils, add metrics framework (#15) Signed-off-by: MrCroxx --- Cargo.lock | 62 +++++++++++++++++++ Cargo.toml | 2 +- foyer-bench/src/main.rs | 17 +++-- foyer-utils/Cargo.toml | 40 ++++++++++++ .../store/utils.rs => foyer-utils/src/bits.rs | 0 .../collections => foyer-utils/src}/dlist.rs | 5 +- .../mod.rs => foyer-utils/src/lib.rs | 3 + foyer/Cargo.toml | 2 + foyer/src/container.rs | 5 +- foyer/src/lib.rs | 50 +++++++++++++-- foyer/src/metrics.rs | 61 ++++++++++++++++++ foyer/src/policies/lru.rs | 3 +- foyer/src/policies/tinylfu.rs | 3 +- foyer/src/store/mod.rs | 14 ++--- 14 files changed, 232 insertions(+), 35 deletions(-) create mode 100644 foyer-utils/Cargo.toml rename foyer/src/store/utils.rs => foyer-utils/src/bits.rs (100%) rename {foyer/src/collections => foyer-utils/src}/dlist.rs (99%) rename foyer/src/collections/mod.rs => foyer-utils/src/lib.rs (93%) create mode 100644 foyer/src/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index cdae3f61..649b2b31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -295,6 +295,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foyer" version = "0.1.0" @@ -305,6 +311,7 @@ dependencies = [ "clap", "cmsketch", "crossbeam", + "foyer-utils", "futures", "hdrhistogram", "itertools", @@ -313,6 +320,7 @@ dependencies = [ "nix", "parking_lot", "paste", + "prometheus", "rand", "rand_mt", "tempfile", @@ -340,6 +348,33 @@ dependencies = [ "tokio", ] +[[package]] +name = "foyer-utils" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "bytesize", + "clap", + "cmsketch", + "crossbeam", + "futures", + "hdrhistogram", + "itertools", + "libc", + "memoffset 0.8.0", + "nix", + "parking_lot", + "paste", + "prometheus", + "rand", + "rand_mt", + "tempfile", + "thiserror", + "tokio", + "twox-hash", +] + [[package]] name = "futures" version = "0.3.28" @@ -516,6 +551,12 @@ dependencies = [ "either", ] +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "libc" version = "0.2.144" @@ -703,6 +744,27 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prometheus" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "protobuf", + "thiserror", +] + +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + [[package]] name = "quote" version = "1.0.27" diff --git a/Cargo.toml b/Cargo.toml index 628cc248..d4e9a2eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] -members = ["foyer", "foyer-bench"] +members = ["foyer", "foyer-bench", "foyer-utils"] [patch.crates-io] # cmsketch = { path = "../cmsketch-rs" } diff --git a/foyer-bench/src/main.rs b/foyer-bench/src/main.rs index 3d2a300a..d5263ed3 100644 --- a/foyer-bench/src/main.rs +++ b/foyer-bench/src/main.rs @@ -27,11 +27,10 @@ use std::time::{Duration, Instant}; use analyze::{analyze, monitor, Metrics}; use clap::Parser; -use foyer::container::{Config, Container}; - -use foyer::store::read_only_file_store::{Config as ReadOnlyFileStoreConfig, ReadOnlyFileStore}; - -use foyer::policies::tinylfu::{Config as TinyLfuConfig, Handle, TinyLfu}; +use foyer::{ + ReadOnlyFileStoreConfig, TinyLfuConfig, TinyLfuReadOnlyFileStoreCache, + TinyLfuReadOnlyFileStoreCacheConfig, +}; use futures::future::join_all; use itertools::Itertools; use rand::rngs::StdRng; @@ -109,9 +108,7 @@ impl Args { assert!(self.pools.is_power_of_two()); } } - -type TContainer = - Container, Handle, Vec, ReadOnlyFileStore>>; +type TContainer = TinyLfuReadOnlyFileStoreCache>; fn is_send_sync_static() {} @@ -159,7 +156,7 @@ async fn main() { tiny_lru_capacity_ratio: 0.01, }; - let config: Config, _, ReadOnlyFileStore<_, Vec>> = Config { + let config = TinyLfuReadOnlyFileStoreCacheConfig { capacity: args.capacity * 1024 * 1024, pool_count_bits: (args.pools as f64).log2() as usize, policy_config, @@ -168,7 +165,7 @@ async fn main() { println!("{:#?}", config); - let container = Container::open(config).await.unwrap(); + let container = TinyLfuReadOnlyFileStoreCache::open(config).await.unwrap(); let container = Arc::new(container); let (iostat_stop_tx, iostat_stop_rx) = oneshot::channel(); diff --git a/foyer-utils/Cargo.toml b/foyer-utils/Cargo.toml new file mode 100644 index 00000000..e1bfd5b7 --- /dev/null +++ b/foyer-utils/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "foyer-utils" +version = "0.1.0" +edition = "2021" +authors = ["MrCroxx "] +description = "Hybrid cache for Rust" +license = "Apache-2.0" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +async-trait = "0.1" +bytes = "1" +cmsketch = "0.1" +crossbeam = "0.8" +futures = "0.3" +itertools = "0.10.5" +libc = "0.2" +memoffset = "0.8" +nix = { version = "0.26", features = ["fs", "mman"] } +parking_lot = "0.12" +paste = "1.0" +prometheus = "0.13" +thiserror = "1" +tokio = { version = "1", features = [ + "rt", + "rt-multi-thread", + "sync", + "macros", + "time", + "signal", +] } +twox-hash = "1" + +[dev-dependencies] +bytesize = "1" +clap = { version = "4", features = ["derive"] } +hdrhistogram = "7" +rand = "0.8.5" +rand_mt = "4.2.1" +tempfile = "3" diff --git a/foyer/src/store/utils.rs b/foyer-utils/src/bits.rs similarity index 100% rename from foyer/src/store/utils.rs rename to foyer-utils/src/bits.rs diff --git a/foyer/src/collections/dlist.rs b/foyer-utils/src/dlist.rs similarity index 99% rename from foyer/src/collections/dlist.rs rename to foyer-utils/src/dlist.rs index 4e5a2ce8..a134ca84 100644 --- a/foyer/src/collections/dlist.rs +++ b/foyer-utils/src/dlist.rs @@ -33,13 +33,14 @@ pub trait Adapter { fn el2en(_: NonNull) -> NonNull; } +#[macro_export] macro_rules! intrusive_dlist { ( $element:ident $(< $( $lt:tt $( : $clt:tt $(+ $dlt:tt)* )? ),+ >)?, $entry:ident, $adapter:ident ) => { struct $adapter; - impl $(< $( $lt $( : $clt $(+ $dlt )* )? ),+ >)? $crate::collections::dlist::Adapter<$element $(< $( $lt ),+ >)?> for $adapter { + impl $(< $( $lt $( : $clt $(+ $dlt )* )? ),+ >)? $crate::dlist::Adapter<$element $(< $( $lt ),+ >)?> for $adapter { fn en2el(entry: std::ptr::NonNull) -> std::ptr::NonNull<$element $(< $( $lt ),+ >)?> { unsafe { let ptr = (entry.as_ptr() as *mut u8) @@ -61,8 +62,6 @@ macro_rules! intrusive_dlist { }; } -pub(crate) use intrusive_dlist; - /// TODO: write docs #[derive(Debug)] pub struct DList> { diff --git a/foyer/src/collections/mod.rs b/foyer-utils/src/lib.rs similarity index 93% rename from foyer/src/collections/mod.rs rename to foyer-utils/src/lib.rs index 4fe9e427..2d58d97e 100644 --- a/foyer/src/collections/mod.rs +++ b/foyer-utils/src/lib.rs @@ -12,4 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![feature(trait_alias)] + +pub mod bits; pub mod dlist; diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml index 69bdec42..720e6ca6 100644 --- a/foyer/Cargo.toml +++ b/foyer/Cargo.toml @@ -12,6 +12,7 @@ async-trait = "0.1" bytes = "1" cmsketch = "0.1" crossbeam = "0.8" +foyer-utils = { path = "../foyer-utils" } futures = "0.3" itertools = "0.10.5" libc = "0.2" @@ -19,6 +20,7 @@ memoffset = "0.8" nix = { version = "0.26", features = ["fs", "mman"] } parking_lot = "0.12" paste = "1.0" +prometheus = "0.13" thiserror = "1" tokio = { version = "1", features = [ "rt", diff --git a/foyer/src/container.rs b/foyer/src/container.rs index eff0be4b..786ac6e4 100644 --- a/foyer/src/container.rs +++ b/foyer/src/container.rs @@ -97,7 +97,7 @@ where .into_iter() .enumerate() .map(|(id, store)| Pool { - id, + _id: id, policy: P::new(config.policy_config.clone()), capacity, size: 0, @@ -175,8 +175,7 @@ where D: Data, S: Store, { - #[allow(unused)] - id: usize, + _id: usize, policy: P, diff --git a/foyer/src/lib.rs b/foyer/src/lib.rs index 47fd517f..3ef71eee 100644 --- a/foyer/src/lib.rs +++ b/foyer/src/lib.rs @@ -21,10 +21,10 @@ pub use policies::Policy; use paste::paste; -pub mod collections; -pub mod container; -pub mod policies; -pub mod store; +mod container; +mod metrics; +mod policies; +mod store; pub trait Weight { fn weight(&self) -> usize; @@ -87,9 +87,47 @@ pub struct WrappedNonNull(pub NonNull); unsafe impl Send for WrappedNonNull {} unsafe impl Sync for WrappedNonNull {} -#[cfg(test)] +// TODO(MrCroxx): Add global error type. +pub type Error = store::error::Error; + +pub type TinyLfuReadOnlyFileStoreCache = container::Container< + I, + policies::tinylfu::TinyLfu, + policies::tinylfu::Handle, + D, + store::read_only_file_store::ReadOnlyFileStore, +>; + +pub type TinyLfuReadOnlyFileStoreCacheConfig = container::Config< + I, + policies::tinylfu::TinyLfu, + policies::tinylfu::Handle, + store::read_only_file_store::ReadOnlyFileStore, +>; + +pub type LruReadOnlyFileStoreCache = container::Container< + I, + policies::lru::Lru, + policies::lru::Handle, + D, + store::read_only_file_store::ReadOnlyFileStore, +>; + +pub type LruReadOnlyFileStoreCacheConfig = container::Config< + I, + policies::lru::Lru, + policies::lru::Handle, + store::read_only_file_store::ReadOnlyFileStore, +>; + +pub use metrics::Metrics; + +pub use policies::lru::Config as LruConfig; +pub use policies::tinylfu::Config as TinyLfuConfig; +pub use store::read_only_file_store::Config as ReadOnlyFileStoreConfig; -pub mod tests { +#[cfg(test)] +mod tests { pub fn is_send_sync_static() {} } diff --git a/foyer/src/metrics.rs b/foyer/src/metrics.rs new file mode 100644 index 00000000..aa84a4cb --- /dev/null +++ b/foyer/src/metrics.rs @@ -0,0 +1,61 @@ +// Copyright 2023 MrCroxx +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use prometheus::{ + register_counter_vec_with_registry, register_histogram_vec_with_registry, + register_int_counter_with_registry, CounterVec, HistogramVec, IntCounter, Registry, +}; + +pub struct Metrics { + pub miss: IntCounter, + + pub latency: HistogramVec, + pub bytes: CounterVec, +} + +impl Default for Metrics { + fn default() -> Self { + Self::new(Registry::new()) + } +} + +impl Metrics { + pub fn new(registry: Registry) -> Self { + let miss = + register_int_counter_with_registry!("foyer_cache_miss", "file cache miss", registry) + .unwrap(); + + let latency = register_histogram_vec_with_registry!( + "foyer_latency", + "foyer latency", + &["op"], + vec![ + 0.0001, 0.001, 0.005, 0.01, 0.02, 0.03, 0.04, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, + 1.0 + ], + registry + ) + .unwrap(); + + let bytes = + register_counter_vec_with_registry!("foyer_bytes", "foyer bytes", &["op"], registry) + .unwrap(); + + Self { + miss, + latency, + bytes, + } + } +} diff --git a/foyer/src/policies/lru.rs b/foyer/src/policies/lru.rs index 50a53353..ebac9999 100644 --- a/foyer/src/policies/lru.rs +++ b/foyer/src/policies/lru.rs @@ -29,7 +29,8 @@ use std::ptr::NonNull; use std::time::SystemTime; -use crate::collections::dlist::{intrusive_dlist, DList, Entry, Iter}; +use foyer_utils::dlist::{DList, Entry, Iter}; +use foyer_utils::intrusive_dlist; use super::Index; diff --git a/foyer/src/policies/tinylfu.rs b/foyer/src/policies/tinylfu.rs index a758a997..e8e44144 100644 --- a/foyer/src/policies/tinylfu.rs +++ b/foyer/src/policies/tinylfu.rs @@ -33,7 +33,8 @@ use std::time::SystemTime; use cmsketch::CMSketchUsize; use twox_hash::XxHash64; -use crate::collections::dlist::{intrusive_dlist, DList, Entry, Iter}; +use foyer_utils::dlist::{DList, Entry, Iter}; +use foyer_utils::intrusive_dlist; use super::Index; diff --git a/foyer/src/store/mod.rs b/foyer/src/store/mod.rs index cbebbaa4..e5d46e20 100644 --- a/foyer/src/store/mod.rs +++ b/foyer/src/store/mod.rs @@ -13,9 +13,9 @@ // limitations under the License. pub mod error; +#[allow(dead_code)] pub mod file; pub mod read_only_file_store; -pub mod utils; use crate::{Data, Index}; use async_trait::async_trait; @@ -62,21 +62,15 @@ pub mod tests { #[derive(Clone, Debug, Default)] pub struct MemoryStore { - pool: usize, inner: Arc>>, } impl MemoryStore { - pub fn new(pool: usize) -> Self { + pub fn new() -> Self { Self { - pool, inner: Arc::new(RwLock::new(HashMap::default())), } } - - pub fn pool(&self) -> usize { - self.pool - } } #[async_trait] @@ -87,8 +81,8 @@ pub mod tests { type C = (); - async fn open(pool: usize, _: Self::C) -> Result { - Ok(Self::new(pool)) + async fn open(_pool: usize, _: Self::C) -> Result { + Ok(Self::new()) } async fn store(&self, index: Self::I, data: Self::D) -> Result<()> {