diff --git a/Cargo.lock b/Cargo.lock
index 64ee2bf0d65..4a5aedcf3c4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1053,11 +1053,14 @@ dependencies = [
  "ciborium",
  "clap",
  "criterion-plot",
+ "csv",
  "is-terminal",
  "itertools 0.10.5",
  "num-traits",
  "once_cell",
  "oorandom",
+ "plotters",
+ "rayon",
  "regex",
  "serde",
  "serde_derive",
@@ -1163,6 +1166,27 @@ dependencies = [
  "typenum",
 ]
 
+[[package]]
+name = "csv"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe"
+dependencies = [
+ "csv-core",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "csv-core"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "ctor"
 version = "0.1.26"
@@ -3540,6 +3564,34 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
 
+[[package]]
+name = "plotters"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
+dependencies = [
+ "plotters-backend",
+]
+
 [[package]]
 name = "portable-atomic"
 version = "1.9.0"
diff --git a/Cargo.toml b/Cargo.toml
index f298946d665..4bab9a48075 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -138,7 +138,7 @@ wasmer = { version = "=5.0.0-rc.1", path = "lib/api", features = [
 	"sys",
 ] }
 anyhow = "1.0"
-criterion = { version = "0.5", default-features = false }
+criterion = { version = "0.5", features = ["csv_output"] }
 clap = { version = "=4.4.11" }
 clap_builder = { version = "=4.4.11" }
 clap_derive = { version = "=4.4.7" }
@@ -229,6 +229,10 @@ opt-level = 3
 [profile.dev.package.digest]
 opt-level = 3
 
+[[bench]]
+name = "compiler_run"
+harness = false
+
 [[bench]]
 name = "deserialize_modules"
 harness = false
diff --git a/benches/compiler_run.rs b/benches/compiler_run.rs
new file mode 100644
index 00000000000..52b6c13371f
--- /dev/null
+++ b/benches/compiler_run.rs
@@ -0,0 +1,77 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use wasmer::*;
+
+static BENCHMARKS_ARTIFACTS_BASE_URL: &str = "https://pub-53a226d993e144159d6f8b993fe0cbf3.r2.dev";
+
+fn get_engine() -> Engine {
+    #[cfg(feature = "llvm")]
+    return LLVM::new().into();
+
+    #[cfg(feature = "singlepass")]
+    return Singlepass::new().into();
+
+    #[cfg(feature = "cranelift")]
+    return Cranelift::new().into();
+
+    #[cfg(not(any(feature = "cranelift", feature = "llvm", feature = "singlepass")))]
+    return Default::default();
+}
+
+pub fn run_fn(c: &mut Criterion, module: &[u8], name: &str, input: i64) {
+    c.bench_function(name, |b| {
+        let engine = get_engine();
+        let mut store = Store::new(engine);
+        let module = Module::new(&store, module).unwrap();
+        let import_object = imports! {};
+        let instance = Instance::new(&mut store, &module, &import_object).unwrap();
+        let func = instance
+            .exports
+            .get_typed_function::<i64, i64>(&store, "run")
+            .unwrap();
+
+        b.iter(|| {
+            func.call(&mut store, input);
+        })
+    });
+}
+
+pub fn download_and_run(c: &mut Criterion) {
+    let name = if cfg!(feature = "cranelift") {
+        "cranelift"
+    } else if cfg!(feature = "llvm") {
+        "llvm"
+    } else if cfg!(feature = "singlepass") {
+        "singlepass"
+    } else if cfg!(feature = "v8") {
+        "v8"
+    } else if cfg!(feature = "wamr") {
+        "wamr"
+    } else if cfg!(feature = "wasmi") {
+        "wasmi"
+    } else {
+        panic!("Unrecognized backend!")
+    };
+
+    let bytes = include_bytes!("./mods/counter.wasm");
+    run_fn(c, bytes, &format!("exec/{name}/counter"), 5_000_000);
+    let bytes = include_bytes!("./mods/primes.wasm");
+    run_fn(c, bytes, &format!("exec/{name}/primes"), 1_000);
+    let bytes = include_bytes!("./mods/fib_rec.wasm");
+    run_fn(c, bytes, &format!("exec/{name}/fib_rec"), 40);
+    let bytes = include_bytes!("./mods/fib_iter.wasm");
+    run_fn(c, bytes, &format!("exec/{name}/fib_iter"), 2_000_000);
+    let bytes = include_bytes!("./mods/bulk_ops.wasm");
+    run_fn(c, bytes, &format!("exec/{name}/bulk_ops"), 5_000);
+    let bytes = include_bytes!("./mods/matmul.wasm");
+    run_fn(c, bytes, &format!("exec/{name}/matmul"), 200);
+    let bytes = include_bytes!("./mods/argon2.wasm");
+    run_fn(c, bytes, &format!("exec/{name}/argon2"), 1);
+}
+
+criterion_group!(
+    name = run_benches;
+    config = Criterion::default().sample_size(60);
+    targets = download_and_run
+);
+
+criterion_main!(run_benches);
diff --git a/benches/mods/argon2.wasm b/benches/mods/argon2.wasm
new file mode 100644
index 00000000000..704cd488c2f
Binary files /dev/null and b/benches/mods/argon2.wasm differ
diff --git a/benches/mods/bulk_ops.wasm b/benches/mods/bulk_ops.wasm
new file mode 100644
index 00000000000..fe27e5600e9
Binary files /dev/null and b/benches/mods/bulk_ops.wasm differ
diff --git a/benches/mods/counter.wasm b/benches/mods/counter.wasm
new file mode 100644
index 00000000000..6551b6de650
Binary files /dev/null and b/benches/mods/counter.wasm differ
diff --git a/benches/mods/fib_iter.wasm b/benches/mods/fib_iter.wasm
new file mode 100644
index 00000000000..9526bb1c209
Binary files /dev/null and b/benches/mods/fib_iter.wasm differ
diff --git a/benches/mods/fib_rec.wasm b/benches/mods/fib_rec.wasm
new file mode 100644
index 00000000000..75d0af323b1
Binary files /dev/null and b/benches/mods/fib_rec.wasm differ
diff --git a/benches/mods/matmul.wasm b/benches/mods/matmul.wasm
new file mode 100644
index 00000000000..3a51f2c9ae0
Binary files /dev/null and b/benches/mods/matmul.wasm differ
diff --git a/benches/mods/primes.wasm b/benches/mods/primes.wasm
new file mode 100644
index 00000000000..6551b6de650
Binary files /dev/null and b/benches/mods/primes.wasm differ