Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(benches): Add compilation benches, use remote artifacts #5236

Merged
merged 3 commits into from
Nov 11, 2024
Merged

feat(benches): Add compilation benches, use remote artifacts #5236

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f145bfe
feat(benches): Add compilation benches, use remote artifacts
xdoardo Nov 8, 2024
c2543cf
fix(benches): Re-enable all compilation benchmarks
xdoardo Nov 8, 2024
9eda678
fix(benches): Don't use `native-tls` and default features from reqwes…
xdoardo Nov 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ tracing-subscriber = { version = "0.3", default-features = false, features = [
"env-filter",
"fmt",
] }
reqwest.workspace = true
reqwest = {workspace = true, features = ["blocking", "rustls-tls"]}

[features]
# Don't add the compiler features in default, please add them on the Makefile
Expand Down Expand Up @@ -220,7 +220,11 @@ opt-level = 3
opt-level = 3

[[bench]]
name = "compiler_run"
name = "run"
harness = false

[[bench]]
name = "compile"
harness = false

[[bench]]
Expand Down
149 changes: 149 additions & 0 deletions benches/compile.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
use criterion::{
criterion_group, criterion_main, measurement::WallTime, BenchmarkGroup, Criterion,
};
use wasmer::*;

static BENCHMARKS_ARTIFACTS_BASE_URL: &str = "https://pub-083d1a0568d446d1aa5b2e07bd16983b.r2.dev";

fn get_engine() -> Engine {
#[cfg(feature = "llvm")]
return LLVM::new().into();

#[cfg(feature = "singlepass")]
return Singlepass::new().into();

#[cfg(feature = "cranelift")]
return Cranelift::new().into();

#[cfg(not(any(feature = "cranelift", feature = "llvm", feature = "singlepass")))]
return Default::default();
}

pub fn compile_wasm(c: &mut BenchmarkGroup<'_, WallTime>, module: &[u8], name: &str) {
c.bench_function(name, |b| {
let engine = get_engine();

b.iter(|| {
let store = Store::new(engine.clone());
assert!(Module::new(&store, module).is_ok());
})
});
}

pub fn download_and_compile_small(c: &mut BenchmarkGroup<'_, WallTime>) {
let name = if cfg!(feature = "cranelift") {
"cranelift"
} else if cfg!(feature = "llvm") {
"llvm"
} else if cfg!(feature = "singlepass") {
"singlepass"
} else if cfg!(feature = "v8") {
"v8"
} else if cfg!(feature = "wamr") {
"wamr"
} else if cfg!(feature = "wasmi") {
"wasmi"
} else {
panic!("Unrecognized backend!")
};

let modules = [
"counter", "primes", "fib_rec", "fib_iter", "bulk_ops", "matmul",
];

for module in modules {
let bytes =
reqwest::blocking::get(format!("{BENCHMARKS_ARTIFACTS_BASE_URL}/{module}.wasm"))
.unwrap()
.bytes()
.unwrap();
compile_wasm(c, bytes.as_ref(), &format!("exec/{name}/{module}"));
}
}

pub fn download_and_compile_medium(c: &mut BenchmarkGroup<'_, WallTime>) {
let name = if cfg!(feature = "cranelift") {
"cranelift"
} else if cfg!(feature = "llvm") {
"llvm"
} else if cfg!(feature = "singlepass") {
"singlepass"
} else if cfg!(feature = "v8") {
"v8"
} else if cfg!(feature = "wamr") {
"wamr"
} else if cfg!(feature = "wasmi") {
"wasmi"
} else {
panic!("Unrecognized backend!")
};

let modules = ["bash", "irb", "argon2"];

for module in modules {
let bytes =
reqwest::blocking::get(format!("{BENCHMARKS_ARTIFACTS_BASE_URL}/{module}.wasm"))
.unwrap()
.bytes()
.unwrap();
compile_wasm(c, bytes.as_ref(), &format!("exec/{name}/{module}"));
}
}

pub fn download_and_compile_large(c: &mut BenchmarkGroup<'_, WallTime>) {
let name = if cfg!(feature = "cranelift") {
"cranelift"
} else if cfg!(feature = "llvm") {
"llvm"
} else if cfg!(feature = "singlepass") {
"singlepass"
} else if cfg!(feature = "v8") {
"v8"
} else if cfg!(feature = "wamr") {
"wamr"
} else if cfg!(feature = "wasmi") {
"wasmi"
} else {
panic!("Unrecognized backend!")
};

let modules = [
"winterjs",
"wasix_axum",
"static_web_server",
"s3_server",
"python",
"php",
];

for module in modules {
let bytes =
reqwest::blocking::get(format!("{BENCHMARKS_ARTIFACTS_BASE_URL}/{module}.wasm"))
.unwrap()
.bytes()
.unwrap();
compile_wasm(c, bytes.as_ref(), &format!("exec/{name}/{module}"));
}
}

pub fn download_and_compile(c: &mut Criterion) {
{
let mut group = c.benchmark_group("large_benches");
group.sample_size(10);
download_and_compile_large(&mut group);
}

{
let mut group = c.benchmark_group("medium_benches");
group.sample_size(40);
download_and_compile_medium(&mut group);
}
{
let mut group = c.benchmark_group("small_benches");
group.sample_size(60);
download_and_compile_small(&mut group);
}
}

criterion_group!(benches, download_and_compile);
criterion_main!(benches);
80 changes: 17 additions & 63 deletions benches/deserialize_modules.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,88 +4,42 @@ use wasmer::*;

static BENCHMARKS_ARTIFACTS_BASE_URL: &str = "https://pub-53a226d993e144159d6f8b993fe0cbf3.r2.dev";

pub fn serialize_deserialize(c: &mut Criterion, small: &[u8], medium: &[u8], large: &[u8]) {
pub fn serialize_deserialize(c: &mut Criterion, bytes: &[u8], name: &str) {
let engine = Engine::default();

c.bench_function(&format!("rkyv_from-binary_small"), |b| {
c.bench_function(&format!("rkyv_from-binary_{name}"), |b| {
b.iter(|| {
let module = Module::from_binary(&engine, small);
assert!(module.is_ok());
drop(module);
})
});

c.bench_function(&format!("rkyv_from-binary_medium"), |b| {
b.iter(|| {
let module = Module::from_binary(&engine, medium);
assert!(module.is_ok());
drop(module);
})
});

c.bench_function(&format!("rkyv_from-binary_large"), |b| {
b.iter(|| {
let module = Module::from_binary(&engine, large);
let module = Module::from_binary(&engine, bytes);
assert!(module.is_ok());
drop(module);
})
});
}

pub fn serialize_deserialize_unchecked(
c: &mut Criterion,
small: &[u8],
medium: &[u8],
large: &[u8],
) {
pub fn serialize_deserialize_unchecked(c: &mut Criterion, bytes: &[u8], name: &str) {
let engine = Engine::default();

c.bench_function(&format!("rkyv_from-binary-unchecked_small"), |b| {
c.bench_function(&format!("rkyv_from-binary-unchecked_{name}"), |b| {
b.iter(|| unsafe {
let module = Module::from_binary_unchecked(&engine, small);
assert!(module.is_ok());
drop(module);
})
});

c.bench_function(&format!("rkyv_from-binary-unchecked_medium"), |b| {
b.iter(|| unsafe {
let module = Module::from_binary_unchecked(&engine, medium);
assert!(module.is_ok());
drop(module);
})
});

c.bench_function(&format!("rkyv_from-binary-unchecked_large"), |b| {
b.iter(|| unsafe {
let module = Module::from_binary_unchecked(&engine, large);
let module = Module::from_binary_unchecked(&engine, bytes);
assert!(module.is_ok());
drop(module);
})
});
}

pub fn download_and_run(c: &mut Criterion) {
let small = reqwest::blocking::get(BENCHMARKS_ARTIFACTS_BASE_URL.to_owned() + "/small.wasm")
.unwrap()
.bytes()
.unwrap();

let medium = reqwest::blocking::get(BENCHMARKS_ARTIFACTS_BASE_URL.to_owned() + "/medium.wasm")
.unwrap()
.bytes()
.unwrap();

let large = reqwest::blocking::get(BENCHMARKS_ARTIFACTS_BASE_URL.to_owned() + "/large.wasm")
.unwrap()
.bytes()
.unwrap();

let small = small.as_ref();
let medium = medium.as_ref();
let large = large.as_ref();
serialize_deserialize(c, small, medium, large);
serialize_deserialize_unchecked(c, small, medium, large);
let modules = ["small", "medium", "large"];

for module in modules {
let bytes =
reqwest::blocking::get(format!("{BENCHMARKS_ARTIFACTS_BASE_URL}/{module}.wasm"))
.unwrap()
.bytes()
.unwrap();
serialize_deserialize(c, &bytes, module);
serialize_deserialize_unchecked(c, &bytes, module);
}
}

criterion_group!(
Expand Down
34 changes: 19 additions & 15 deletions benches/compiler_run.rs → benches/run.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use criterion::{criterion_group, criterion_main, Criterion};
use wasmer::*;

static BENCHMARKS_ARTIFACTS_BASE_URL: &str = "https://pub-53a226d993e144159d6f8b993fe0cbf3.r2.dev";
static BENCHMARKS_ARTIFACTS_BASE_URL: &str = "https://pub-083d1a0568d446d1aa5b2e07bd16983b.r2.dev";

fn get_engine() -> Engine {
#[cfg(feature = "llvm")]
Expand Down Expand Up @@ -52,20 +52,24 @@ pub fn download_and_run(c: &mut Criterion) {
panic!("Unrecognized backend!")
};

let bytes = include_bytes!("./mods/counter.wasm");
run_fn(c, bytes, &format!("exec/{name}/counter"), 5_000_000);
let bytes = include_bytes!("./mods/primes.wasm");
run_fn(c, bytes, &format!("exec/{name}/primes"), 1_000);
let bytes = include_bytes!("./mods/fib_rec.wasm");
run_fn(c, bytes, &format!("exec/{name}/fib_rec"), 40);
let bytes = include_bytes!("./mods/fib_iter.wasm");
run_fn(c, bytes, &format!("exec/{name}/fib_iter"), 2_000_000);
let bytes = include_bytes!("./mods/bulk_ops.wasm");
run_fn(c, bytes, &format!("exec/{name}/bulk_ops"), 5_000);
let bytes = include_bytes!("./mods/matmul.wasm");
run_fn(c, bytes, &format!("exec/{name}/matmul"), 200);
let bytes = include_bytes!("./mods/argon2.wasm");
run_fn(c, bytes, &format!("exec/{name}/argon2"), 1);
let modules = [
("counter", 5_000_000),
("primes", 1_000),
("fib_rec", 40),
("fib_iter", 2_000_000),
("bulk_ops", 5_000),
("matmul", 200),
("argon2", 1),
];

for (module, arg) in modules {
let bytes =
reqwest::blocking::get(format!("{BENCHMARKS_ARTIFACTS_BASE_URL}/{module}.wasm"))
.unwrap()
.bytes()
.unwrap();
run_fn(c, bytes.as_ref(), &format!("exec/{name}/{module}"), arg);
}
}

criterion_group!(
Expand Down
Loading