Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 3 additions & 7 deletions .clippy.toml
Original file line number Diff line number Diff line change
Expand Up @@ -45,17 +45,13 @@ path = "tempfile::NamedTempFile::new"
reason = """The temporary files created by this method are not persistable if the temporary directory lives on a different filesystem than the target directory. While it is valid in other contexts (if not persisting files), it was misused many times and so we are banning it. Consider using `tempfile::NamedTempFile::new_in` or `tempfile::NamedTempFile::Builder"""

[[disallowed-methods]]
path = "lru::LruCache::new"
path = "hashlink::LruCache::new"
reason = """Use SizeTrackingLruCache instead."""

[[disallowed-methods]]
path = "lru::LruCache::with_hasher"
path = "hashlink::LruCache::with_hasher"
reason = """Use SizeTrackingLruCache instead."""

[[disallowed-methods]]
path = "lru::LruCache::unbounded"
reason = """Avoid unbounded lru cache for potential memory leak, use SizeTrackingLruCache instead."""

[[disallowed-methods]]
path = "lru::LruCache::unbounded_with_hasher"
path = "hashlink::LruCache::new_unbounded"
reason = """Avoid unbounded lru cache for potential memory leak, use SizeTrackingLruCache instead."""
30 changes: 16 additions & 14 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 4 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ cid = { version = "0.11", default-features = false, features = ["std"] }
dhat = "0.3"
flume = "0.11"
futures = "0.3"
get-size2 = { version = "0.6", features = ["derive"] }
get-size2 = { version = "0.6", features = ["derive", "hashbrown"] }
hashlink = "0.10"
libp2p = { version = "0.56", default-features = false }
libp2p-swarm-test = { version = "0.6", default-features = false, features = ["tokio"] }
multihash-codetable = { version = "0.1", features = ["blake2b", "blake2s", "blake3", "sha2", "sha3", "strobe"] }
Expand Down Expand Up @@ -98,6 +99,8 @@ get-size2 = { workspace = true }
gethostname = "1"
git-version = "0.3"
group = "0.13"
hashbrown = "0.15"
hashlink = { workspace = true }
hex = { version = "0.4", features = ["serde"] }
hickory-resolver = { version = "0.25", default-features = false, features = ["system-config", "tokio"] }
http = "1"
Expand Down Expand Up @@ -139,7 +142,6 @@ libp2p = { workspace = true, features = [
'secp256k1',
] }
libsecp256k1 = "0.7"
lru = "0.16"
md5 = { package = "md-5", version = "0.10" }
memmap2 = "0.9"
memory-stats = "1"
Expand Down
4 changes: 4 additions & 0 deletions benches/car-index.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
// Copyright 2019-2025 ChainSafe Systems
// SPDX-License-Identifier: Apache-2.0, MIT

//! ```console
//! $ cargo bench --bench car-index
//! ```

use cid::Cid;
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
use forest::benchmark_private::{
Expand Down
1 change: 1 addition & 0 deletions benches/example-benchmark.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// Copyright 2019-2025 ChainSafe Systems
// SPDX-License-Identifier: Apache-2.0, MIT

//! ```console
//! $ cargo bench --bench example-benchmark
//! ```
Expand Down
6 changes: 3 additions & 3 deletions src/db/car/forest.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@ use crate::db::car::RandomAccessFileReader;
use crate::db::car::plain::write_skip_frame_header_async;
use crate::utils::db::car_stream::{CarBlock, CarV1Header};
use crate::utils::encoding::from_slice_with_fallback;
use crate::utils::get_size::CidWrapper;
use crate::utils::io::EitherMmapOrRandomAccessFile;
use ahash::{HashMap, HashMapExt};
use byteorder::LittleEndian;
use bytes::{BufMut as _, Bytes, BytesMut, buf::Writer};
use cid::Cid;
Expand Down Expand Up @@ -241,14 +241,14 @@ where
let cursor = Cursor::new_pos(entire_file, position);
let mut zstd_frame = decode_zstd_single_frame(cursor)?;
// Parse all key-value pairs and insert them into a map
let mut block_map = HashMap::new();
let mut block_map = hashbrown::HashMap::new();
while let Some(block_frame) =
UviBytes::<Bytes>::default().decode_eof(&mut zstd_frame)?
{
let CarBlock { cid, data } = CarBlock::from_bytes(block_frame)?;
block_map.insert(cid.into(), data);
}
let get_result = block_map.get(&(*k).into()).cloned();
let get_result = block_map.get(&CidWrapper::from(*k)).cloned();
self.frame_cache.put(position, self.cache_key, block_map);

// This lookup only fails in case of a hash collision
Expand Down
23 changes: 15 additions & 8 deletions src/db/car/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ use get_size2::GetSize as _;
pub use many::ManyCar;
pub use plain::PlainCar;

use ahash::HashMap;
use cid::Cid;
use positioned_io::{ReadAt, Size};
use std::{
Expand Down Expand Up @@ -58,7 +57,9 @@ pub struct ZstdFrameCache {
/// cache exceeds this amount.
pub max_size: usize,
current_size: AtomicUsize,
lru: SizeTrackingLruCache<(FrameOffset, CacheKey), HashMap<CidWrapper, Vec<u8>>>,
// use `hashbrown::HashMap` here because its `GetSize` implementation is accurate
// (thanks to `hashbrown::HashMap::allocation_size`).
lru: SizeTrackingLruCache<(FrameOffset, CacheKey), hashbrown::HashMap<CidWrapper, Vec<u8>>>,
}

impl Default for ZstdFrameCache {
Expand All @@ -83,11 +84,18 @@ impl ZstdFrameCache {
.cache()
.write()
.get(&(offset, key))
.map(|index| index.get(&cid.into()).cloned())
.map(|index| index.get(&CidWrapper::from(cid)).cloned())
}

/// Insert entry into lru-cache and evict pages if `max_size` has been exceeded.
pub fn put(&self, offset: FrameOffset, key: CacheKey, index: HashMap<CidWrapper, Vec<u8>>) {
pub fn put(
&self,
offset: FrameOffset,
key: CacheKey,
mut index: hashbrown::HashMap<CidWrapper, Vec<u8>>,
) {
index.shrink_to_fit();

let lru_key = (offset, key);
let lru_key_size = lru_key.get_size();
let entry_size = index.get_size();
Expand All @@ -96,7 +104,7 @@ impl ZstdFrameCache {
return;
}

if let Some((_, prev_entry)) = self.lru.push(lru_key, index) {
if let Some(prev_entry) = self.lru.push(lru_key, index) {
// keys are cancelled out
self.current_size.fetch_add(entry_size, Ordering::Relaxed);
self.current_size
Expand All @@ -122,7 +130,6 @@ impl ZstdFrameCache {
mod tests {
use super::*;
use crate::utils::{multihash::MultihashCode, rand::forest_rng};
use ahash::HashMap;
use fvm_ipld_encoding::IPLD_RAW;
use multihash_derive::MultihashDigest;
use rand::Rng;
Expand All @@ -147,8 +154,8 @@ mod tests {
}
}

fn gen_index(rng: &mut impl Rng) -> HashMap<CidWrapper, Vec<u8>> {
let mut map = HashMap::default();
fn gen_index(rng: &mut impl Rng) -> hashbrown::HashMap<CidWrapper, Vec<u8>> {
let mut map = hashbrown::HashMap::default();
for _ in 0..10 {
let vec_len = rng.gen_range(64..1024);
let mut data = vec![0; vec_len];
Expand Down
17 changes: 9 additions & 8 deletions src/utils/cache/lru.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use std::{
};

use get_size2::GetSize;
use lru::LruCache;
use hashlink::LruCache;
use parking_lot::RwLock;
use prometheus_client::{
collector::Collector,
Expand Down Expand Up @@ -68,9 +68,10 @@ where
#[allow(clippy::disallowed_methods)]
cache: Arc::new(RwLock::new(
capacity
.map(From::from)
.map(LruCache::new)
// For constructing lru cache that is bounded by memory usage instead of length
.unwrap_or_else(LruCache::unbounded),
.unwrap_or_else(LruCache::new_unbounded),
)),
}
}
Expand Down Expand Up @@ -120,16 +121,16 @@ where
&self.cache
}

pub fn push(&self, k: K, v: V) -> Option<(K, V)> {
self.cache.write().push(k, v)
pub fn push(&self, k: K, v: V) -> Option<V> {
self.cache.write().insert(k, v)
}

pub fn contains<Q>(&self, k: &Q) -> bool
where
K: Borrow<Q>,
Q: Hash + Eq + ?Sized,
{
self.cache.read().contains(k)
self.cache.read().contains_key(k)
}

pub fn get_cloned<Q>(&self, k: &Q) -> Option<V>
Expand All @@ -149,15 +150,15 @@ where
}

pub fn pop_lru(&self) -> Option<(K, V)> {
self.cache.write().pop_lru()
self.cache.write().remove_lru()
}

pub fn len(&self) -> usize {
self.cache.read().len()
}

pub fn cap(&self) -> usize {
self.cache.read().cap().get()
self.cache.read().capacity()
}

pub(crate) fn size_in_bytes(&self) -> usize {
Expand Down Expand Up @@ -215,7 +216,7 @@ where
let cap_metric_help =
format!("Capacity of LruCache {}_{}", self.cache_name, self.cache_id);
let cap: Gauge = Default::default();
cap.set(self.cache.read().cap().get() as _);
cap.set(self.cap() as _);
let cap_metric_encoder = encoder.encode_descriptor(
&cap_metric_name,
&cap_metric_help,
Expand Down
Loading