Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: use bytesize for foyer-bench #698

Merged
merged 4 commits into from
Sep 10, 2024
Merged

refactor: use bytesize for foyer-bench #698

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
852c41c
refactor: use bytesize for foyer-bench
MrCroxx Sep 10, 2024
154035f
chore: fix ci
MrCroxx Sep 10, 2024
366d7a8
chore: switch bytesize repo to foyer-bytesize
MrCroxx Sep 10, 2024
b981f90
fix: fix writer and reader units
MrCroxx Sep 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .github/template/template.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ jobs:
CI: true
run: |
mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench/codecov
cargo llvm-cov --no-report run --package foyer-bench --bin foyer-bench --features "strict_assertions,sanity" -- --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/codecov --mem 16 --disk 256 --region-size 16 --get-range 1000 --w-rate 1 --r-rate 1 --admission-rate-limit 10 --time 60
cargo llvm-cov --no-report run --package foyer-bench --bin foyer-bench --features "strict_assertions,sanity" -- --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/codecov --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60
- name: Generate codecov report
run: |
cargo llvm-cov report --lcov --output-path lcov.info
Expand Down Expand Up @@ -231,7 +231,7 @@ jobs:
run: |-
cargo build --all --features deadlock
mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage/deadlock
timeout 2m ./target/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/deadlock --mem 16 --disk 256 --region-size 16 --get-range 1000 --w-rate 1 --r-rate 1 --admission-rate-limit 10 --time 60
timeout 2m ./target/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/deadlock --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60
asan:
name: run with address saniziter
runs-on: ubuntu-latest
Expand Down Expand Up @@ -269,7 +269,7 @@ jobs:
run: |-
cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench/asan
timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/asan --mem 16 --disk 256 --region-size 16 --get-range 1000 --w-rate 1 --r-rate 1 --admission-rate-limit 10 --time 60
timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/asan --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60
- name: Prepare Artifacts on Failure
if: ${{ failure() }}
run: |-
Expand Down Expand Up @@ -317,7 +317,7 @@ jobs:
run: |-
cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench/lsan
timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/lsan --mem 16 --disk 256 --region-size 16 --get-range 1000 --w-rate 1 --r-rate 1 --admission-rate-limit 10 --time 60
timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/lsan --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60
- name: Prepare Artifacts on Failure
if: ${{ failure() }}
run: |-
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/main.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ jobs:
CI: true
run: |
mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench/codecov
cargo llvm-cov --no-report run --package foyer-bench --bin foyer-bench --features "strict_assertions,sanity" -- --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/codecov --mem 16 --disk 256 --region-size 16 --get-range 1000 --w-rate 1 --r-rate 1 --admission-rate-limit 10 --time 60
cargo llvm-cov --no-report run --package foyer-bench --bin foyer-bench --features "strict_assertions,sanity" -- --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/codecov --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60
- name: Generate codecov report
run: |
cargo llvm-cov report --lcov --output-path lcov.info
Expand Down Expand Up @@ -237,7 +237,7 @@ jobs:
run: |-
cargo build --all --features deadlock
mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage/deadlock
timeout 2m ./target/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/deadlock --mem 16 --disk 256 --region-size 16 --get-range 1000 --w-rate 1 --r-rate 1 --admission-rate-limit 10 --time 60
timeout 2m ./target/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/deadlock --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60
asan:
name: run with address saniziter
runs-on: ubuntu-latest
Expand Down Expand Up @@ -275,7 +275,7 @@ jobs:
run: |-
cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench/asan
timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/asan --mem 16 --disk 256 --region-size 16 --get-range 1000 --w-rate 1 --r-rate 1 --admission-rate-limit 10 --time 60
timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/asan --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60
- name: Prepare Artifacts on Failure
if: ${{ failure() }}
run: |-
Expand Down Expand Up @@ -323,7 +323,7 @@ jobs:
run: |-
cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench/lsan
timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/lsan --mem 16 --disk 256 --region-size 16 --get-range 1000 --w-rate 1 --r-rate 1 --admission-rate-limit 10 --time 60
timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/lsan --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60
- name: Prepare Artifacts on Failure
if: ${{ failure() }}
run: |-
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/pull-request.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ jobs:
CI: true
run: |
mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench/codecov
cargo llvm-cov --no-report run --package foyer-bench --bin foyer-bench --features "strict_assertions,sanity" -- --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/codecov --mem 16 --disk 256 --region-size 16 --get-range 1000 --w-rate 1 --r-rate 1 --admission-rate-limit 10 --time 60
cargo llvm-cov --no-report run --package foyer-bench --bin foyer-bench --features "strict_assertions,sanity" -- --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/codecov --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60
- name: Generate codecov report
run: |
cargo llvm-cov report --lcov --output-path lcov.info
Expand Down Expand Up @@ -236,7 +236,7 @@ jobs:
run: |-
cargo build --all --features deadlock
mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage/deadlock
timeout 2m ./target/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/deadlock --mem 16 --disk 256 --region-size 16 --get-range 1000 --w-rate 1 --r-rate 1 --admission-rate-limit 10 --time 60
timeout 2m ./target/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/deadlock --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60
asan:
name: run with address saniziter
runs-on: ubuntu-latest
Expand Down Expand Up @@ -274,7 +274,7 @@ jobs:
run: |-
cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench/asan
timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/asan --mem 16 --disk 256 --region-size 16 --get-range 1000 --w-rate 1 --r-rate 1 --admission-rate-limit 10 --time 60
timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/asan --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60
- name: Prepare Artifacts on Failure
if: ${{ failure() }}
run: |-
Expand Down Expand Up @@ -322,7 +322,7 @@ jobs:
run: |-
cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench/lsan
timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/lsan --mem 16 --disk 256 --region-size 16 --get-range 1000 --w-rate 1 --r-rate 1 --admission-rate-limit 10 --time 60
timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/lsan --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60
- name: Prepare Artifacts on Failure
if: ${{ failure() }}
run: |-
Expand Down
1 change: 1 addition & 0 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ fastrace = "0.7"
fastrace-jaeger = "0.7"
fastrace-opentelemetry = "0.7"
clap = { version = "4", features = ["derive"] }
bytesize = { package = "foyer-bytesize", version = "2" }

[profile.release]
debug = true
2 changes: 1 addition & 1 deletion foyer-bench/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ readme = "../README.md"

[dependencies]
anyhow = "1"
bytesize = "1"
bytesize = { workspace = true }
clap = { workspace = true }
console-subscriber = { version = "0.4", optional = true }
fastrace = { workspace = true, optional = true }
Expand Down
88 changes: 44 additions & 44 deletions foyer-bench/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
};

use analyze::{analyze, monitor, Metrics};
use bytesize::MIB;
use bytesize::ByteSize;
use clap::{builder::PossibleValuesParser, ArgGroup, Parser};
use foyer::{
Compression, DirectFileDeviceOptionsBuilder, DirectFsDeviceOptionsBuilder, FifoConfig, FifoPicker, HybridCache,
Expand Down Expand Up @@ -73,13 +73,13 @@
#[arg(short, long)]
dir: Option<String>,

/// In-memory cache capacity. (MiB)
#[arg(long, default_value_t = 1024)]
mem: usize,
/// In-memory cache capacity.
#[arg(long, default_value_t = ByteSize::gib(1))]
mem: ByteSize,

Check warning on line 78 in foyer-bench/src/main.rs

View check run for this annotation

Codecov / codecov/patch

foyer-bench/src/main.rs#L78 

Added line #L78 was not covered by tests

/// Disk cache capacity. (MiB)
#[arg(long, default_value_t = 1024)]
disk: usize,
/// Disk cache capacity.
#[arg(long, default_value_t = ByteSize::gib(1))]
disk: ByteSize,

Check warning on line 82 in foyer-bench/src/main.rs

View check run for this annotation

Codecov / codecov/patch

foyer-bench/src/main.rs#L82 

Added line #L82 was not covered by tests

/// (s)
#[arg(short, long, default_value_t = 60)]
Expand All @@ -89,29 +89,29 @@
#[arg(long, default_value_t = 2)]
report_interval: u64,

/// Write rate limit per writer. (MiB)
#[arg(long, default_value_t = 0.0)]
w_rate: f64,
/// Write rate limit per writer.
#[arg(long, default_value_t = ByteSize::b(0))]
w_rate: ByteSize,

Check warning on line 94 in foyer-bench/src/main.rs

View check run for this annotation

Codecov / codecov/patch

foyer-bench/src/main.rs#L94 

Added line #L94 was not covered by tests

/// Read rate limit per reader. (MiB)
#[arg(long, default_value_t = 0.0)]
r_rate: f64,
/// Read rate limit per reader.
#[arg(long, default_value_t = ByteSize::b(0))]
r_rate: ByteSize,

Check warning on line 98 in foyer-bench/src/main.rs

View check run for this annotation

Codecov / codecov/patch

foyer-bench/src/main.rs#L98 

Added line #L98 was not covered by tests

/// Min entry size (B).
#[arg(long, default_value_t = 64 * 1024)]
entry_size_min: usize,
/// Min entry size.
#[arg(long, default_value_t = ByteSize::kib(64))]
entry_size_min: ByteSize,

Check warning on line 102 in foyer-bench/src/main.rs

View check run for this annotation

Codecov / codecov/patch

foyer-bench/src/main.rs#L102 

Added line #L102 was not covered by tests

/// Max entry size (B).
#[arg(long, default_value_t = 64 * 1024)]
entry_size_max: usize,
/// Max entry size.
#[arg(long, default_value_t = ByteSize::kib(64))]
entry_size_max: ByteSize,

Check warning on line 106 in foyer-bench/src/main.rs

View check run for this annotation

Codecov / codecov/patch

foyer-bench/src/main.rs#L106 

Added line #L106 was not covered by tests

/// Reader lookup key range.
#[arg(long, default_value_t = 10000)]
get_range: u64,

/// Disk cache region size. (MiB)
#[arg(long, default_value_t = 64)]
region_size: usize,
/// Disk cache region size.
#[arg(long, default_value_t = ByteSize::mib(64))]
region_size: ByteSize,

Check warning on line 114 in foyer-bench/src/main.rs

View check run for this annotation

Codecov / codecov/patch

foyer-bench/src/main.rs#L114 

Added line #L114 was not covered by tests

/// Flusher count.
#[arg(long, default_value_t = 4)]
Expand All @@ -136,13 +136,13 @@
#[arg(long, default_value_t = 16)]
recover_concurrency: usize,

/// Enable rated ticket admission picker if `admission_rate_limit > 0`. (MiB/s)
#[arg(long, default_value_t = 0)]
admission_rate_limit: usize,
/// Enable rated ticket admission picker if `admission_rate_limit > 0`.
#[arg(long, default_value_t = ByteSize::b(0))]
admission_rate_limit: ByteSize,

Check warning on line 141 in foyer-bench/src/main.rs

View check run for this annotation

Codecov / codecov/patch

foyer-bench/src/main.rs#L141 

Added line #L141 was not covered by tests

/// Enable rated ticket reinsertion picker if `reinsertion_rate_limit > 0`. (MiB/s)
#[arg(long, default_value_t = 0)]
reinsertion_rate_limit: usize,
/// Enable rated ticket reinsertion picker if `reinsertion_rate_limit > 0`.
#[arg(long, default_value_t = ByteSize::b(0))]
reinsertion_rate_limit: ByteSize,

Check warning on line 145 in foyer-bench/src/main.rs

View check run for this annotation

Codecov / codecov/patch

foyer-bench/src/main.rs#L145 

Added line #L145 was not covered by tests

/// `0` means use default.
#[arg(long, default_value_t = 0)]
Expand Down Expand Up @@ -264,14 +264,14 @@
"uniform" => {
// interval = 1 / freq = 1 / (rate / size) = size / rate
let interval =
((args.entry_size_min + args.entry_size_max) >> 1) as f64 / (args.w_rate * 1024.0 * 1024.0);
((args.entry_size_min + args.entry_size_max).as_u64() >> 1) as f64 / (args.w_rate.as_u64() as f64);

Check warning on line 267 in foyer-bench/src/main.rs

View check run for this annotation

Codecov / codecov/patch

foyer-bench/src/main.rs#L267 

Added line #L267 was not covered by tests
let interval = Duration::from_secs_f64(interval);
TimeSeriesDistribution::Uniform { interval }
}
"zipf" => {
// interval = 1 / freq = 1 / (rate / size) = size / rate
let interval =
((args.entry_size_min + args.entry_size_max) >> 1) as f64 / (args.w_rate * 1024.0 * 1024.0);
((args.entry_size_min + args.entry_size_max).as_u64() >> 1) as f64 / (args.w_rate.as_u64() as f64);

Check warning on line 274 in foyer-bench/src/main.rs

View check run for this annotation

Codecov / codecov/patch

foyer-bench/src/main.rs#L274 

Added line #L274 was not covered by tests
let interval = Duration::from_secs_f64(interval);
display_zipf_sample(args.distribution_zipf_n, args.distribution_zipf_s);
TimeSeriesDistribution::Zipf {
Expand Down Expand Up @@ -431,7 +431,7 @@

let builder = HybridCacheBuilder::new()
.with_tracing_config(tracing_config)
.memory(args.mem * MIB as usize)
.memory(args.mem.as_u64() as _)
.with_shards(args.shards);

let builder = match args.eviction.as_str() {
Expand All @@ -453,14 +453,14 @@
builder = match (args.file.as_ref(), args.dir.as_ref()) {
(Some(file), None) => builder.with_device_config(
DirectFileDeviceOptionsBuilder::new(file)
.with_capacity(args.disk * MIB as usize)
.with_region_size(args.region_size * MIB as usize)
.with_capacity(args.disk.as_u64() as _)
.with_region_size(args.region_size.as_u64() as _)

Check warning on line 457 in foyer-bench/src/main.rs

View check run for this annotation

Codecov / codecov/patch

foyer-bench/src/main.rs#L456-L457 

Added lines #L456 - L457 were not covered by tests
.build(),
),
(None, Some(dir)) => builder.with_device_config(
DirectFsDeviceOptionsBuilder::new(dir)
.with_capacity(args.disk * MIB as usize)
.with_file_size(args.region_size * MIB as usize)
.with_capacity(args.disk.as_u64() as _)
.with_file_size(args.region_size.as_u64() as _)
.build(),
),
_ => unreachable!(),
Expand Down Expand Up @@ -497,13 +497,13 @@
_ => unreachable!(),
});

if args.admission_rate_limit > 0 {
if args.admission_rate_limit.as_u64() > 0 {
builder =
builder.with_admission_picker(Arc::new(RateLimitPicker::new(args.admission_rate_limit * MIB as usize)));
builder.with_admission_picker(Arc::new(RateLimitPicker::new(args.admission_rate_limit.as_u64() as _)));
}
if args.reinsertion_rate_limit > 0 {
if args.reinsertion_rate_limit.as_u64() > 0 {
builder =
builder.with_reinsertion_picker(Arc::new(RateLimitPicker::new(args.admission_rate_limit * MIB as usize)));
builder.with_reinsertion_picker(Arc::new(RateLimitPicker::new(args.admission_rate_limit.as_u64() as _)));

Check warning on line 506 in foyer-bench/src/main.rs

View check run for this annotation

Codecov / codecov/patch

foyer-bench/src/main.rs#L506 

Added line #L506 was not covered by tests
}

if args.clean_region_threshold > 0 {
Expand Down Expand Up @@ -570,15 +570,15 @@
}

async fn bench(args: Args, hybrid: HybridCache<u64, Value>, metrics: Metrics, stop_tx: broadcast::Sender<()>) {
let w_rate = if args.w_rate == 0.0 {
let w_rate = if args.w_rate.as_u64() == 0 {
None
} else {
Some(args.w_rate * 1024.0 * 1024.0)
Some(args.w_rate.as_u64() as _)
};
let r_rate = if args.r_rate == 0.0 {
let r_rate = if args.r_rate.as_u64() == 0 {
None
} else {
Some(args.r_rate * 1024.0 * 1024.0)
Some(args.r_rate.as_u64() as _)
};

let counts = (0..args.writers).map(|_| AtomicU64::default()).collect_vec();
Expand All @@ -590,7 +590,7 @@
r_rate,
get_range: args.get_range,
counts,
entry_size_range: args.entry_size_min..args.entry_size_max + 1,
entry_size_range: args.entry_size_min.as_u64() as usize..args.entry_size_max.as_u64() as usize + 1,
time: Duration::from_secs(args.time),
warm_up: Duration::from_secs(args.warm_up),
distribution,
Expand Down
2 changes: 1 addition & 1 deletion foyer-cli/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ readme = "../README.md"

[dependencies]
anyhow = "1"
bytesize = "1"
bytesize = { workspace = true }
clap = { workspace = true }
thiserror = "1"

Expand Down
Loading