From 3c89c0b69ee5dd9afa6d68a4f7bddaae9da45a03 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 6 Mar 2024 17:37:42 +0100
Subject: [PATCH 01/33] feat: add wasmer-argus testsuite

---
 Cargo.lock                               |  59 +++++
 Cargo.toml                               |   1 +
 tests/wasmer-argus/Cargo.toml            |  37 +++
 tests/wasmer-argus/src/argus/config.rs   |  78 ++++++
 tests/wasmer-argus/src/argus/mod.rs      | 302 +++++++++++++++++++++++
 tests/wasmer-argus/src/argus/packages.rs | 222 +++++++++++++++++
 tests/wasmer-argus/src/argus/result.rs   |  51 ++++
 tests/wasmer-argus/src/lib.rs            |   4 +
 tests/wasmer-argus/src/main.rs           |  15 ++
 9 files changed, 769 insertions(+)
 create mode 100644 tests/wasmer-argus/Cargo.toml
 create mode 100644 tests/wasmer-argus/src/argus/config.rs
 create mode 100644 tests/wasmer-argus/src/argus/mod.rs
 create mode 100644 tests/wasmer-argus/src/argus/packages.rs
 create mode 100644 tests/wasmer-argus/src/argus/result.rs
 create mode 100644 tests/wasmer-argus/src/lib.rs
 create mode 100644 tests/wasmer-argus/src/main.rs

diff --git a/Cargo.lock b/Cargo.lock
index 047b0c2d713..43efe51baec 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -737,6 +737,12 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2"
 
+[[package]]
+name = "convert_case"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"
+
 [[package]]
 name = "cooked-waker"
 version = "5.0.0"
@@ -1275,8 +1281,10 @@ version = "0.99.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321"
 dependencies = [
+ "convert_case",
  "proc-macro2",
  "quote",
+ "rustc_version 0.4.0",
  "syn 1.0.109",
 ]
 
@@ -3254,6 +3262,33 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd"
 
+[[package]]
+name = "paw"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09c0fc9b564dbc3dc2ed7c92c0c144f4de340aa94514ce2b446065417c4084e9"
+dependencies = [
+ "paw-attributes",
+ "paw-raw",
+]
+
+[[package]]
+name = "paw-attributes"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f35583365be5d148e959284f42526841917b7bfa09e2d1a7ad5dde2cf0eaa39"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "paw-raw"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f0b59668fe80c5afe998f0c0bf93322bf2cd66cafeeb80581f291716f3467f2"
+
 [[package]]
 name = "pbkdf2"
 version = "0.12.2"
@@ -6106,6 +6141,30 @@ dependencies = [
  "webc",
 ]
 
+[[package]]
+name = "wasmer-argus"
+version = "4.2.6"
+dependencies = [
+ "anyhow",
+ "clap",
+ "cynic",
+ "derive_more",
+ "futures 0.3.30",
+ "indicatif",
+ "log",
+ "paw",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "tokio 1.36.0",
+ "tracing",
+ "tracing-subscriber",
+ "url",
+ "wasmer",
+ "wasmer-api",
+ "webc",
+]
+
 [[package]]
 name = "wasmer-bin-fuzz"
 version = "0.0.0"
diff --git a/Cargo.toml b/Cargo.toml
index 4088c811859..5b18e517cb9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -70,6 +70,7 @@ members = [
     "tests/lib/compiler-test-derive",
     "tests/lib/wast",
     "tests/wasi-wast",
+    "tests/wasmer-argus",
 ]
 resolver = "2"
 
diff --git a/tests/wasmer-argus/Cargo.toml b/tests/wasmer-argus/Cargo.toml
new file mode 100644
index 00000000000..f6d1feadff6
--- /dev/null
+++ b/tests/wasmer-argus/Cargo.toml
@@ -0,0 +1,37 @@
+[package]
+name = "wasmer-argus"
+authors.workspace = true
+edition.workspace = true
+homepage.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+[lib]
+path = "src/lib.rs"
+
+[[bin]]
+name = "wasmer-argus"
+path = "src/main.rs"
+
+[dependencies]
+indicatif = "0.17.8"
+# structopt = { version = "0.2", features = [ "paw" ] }
+paw = "1.0"
+wasmer-api = { version = "0.0.23", path = "../../lib/backend-api" }
+anyhow = "1.0.80"
+log = "0.4.21"
+cynic = "3.4.3"
+url = "2.5.0"
+futures = "0.3.30"
+tracing = "0.1.40"
+tokio = { version = "1.36.0", features = ["rt-multi-thread", "sync", "time", "fs"] }
+clap = {version = "4.4.11", features = ["derive", "string"]}
+tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+serde = { version = "1.0.197", features = ["derive"] }
+serde_json = "1.0.114"
+wasmer = { version = "4.2.6", path = "../../lib/api", features = ["engine", "core", "singlepass", "cranelift"] }
+reqwest = "0.11.24"
+derive_more = "0.99.17"
+webc.workspace = true
diff --git a/tests/wasmer-argus/src/argus/config.rs b/tests/wasmer-argus/src/argus/config.rs
new file mode 100644
index 00000000000..c72fcdf516f
--- /dev/null
+++ b/tests/wasmer-argus/src/argus/config.rs
@@ -0,0 +1,78 @@
+use clap::ValueEnum;
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+use wasmer::{sys::Features, Engine, NativeEngineExt, Target};
+
+fn get_default_out_path() -> PathBuf {
+    let mut path = std::env::current_dir().unwrap();
+    path.push("out");
+    path
+}
+
+fn get_default_token() -> String {
+    std::env::var("WASMER_TOKEN").unwrap_or(String::new())
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, ValueEnum, derive_more::Display)]
+pub enum Backend {
+    LLVM,
+    Singlepass,
+    Cranelift,
+}
+
+impl Backend {
+    pub fn to_engine(&self) -> Engine {
+        match self {
+            Backend::LLVM => todo!(),
+            Backend::Singlepass => Engine::new(
+                Box::new(wasmer::Singlepass::new()),
+                Target::default(),
+                Features::default(),
+            ),
+            Backend::Cranelift => Engine::new(
+                Box::new(wasmer::Cranelift::new()),
+                Target::default(),
+                Features::default(),
+            ),
+        }
+    }
+}
+
+/// Fetch and test packages from a WebContainer registry.
+#[derive(Debug, clap::Parser, Clone, Serialize, Deserialize)]
+#[command(version, about, long_about = None)]
+pub struct ArgusConfig {
+    /// The GraphQL endpoint of the registry to test
+    #[arg(
+        short,
+        long,
+        default_value_t = String::from("http://registry.wasmer.io/graphql")
+    )]
+    pub registry_url: String,
+
+    /// The backend to test the compilation against
+    #[arg(short = 'b', long = "backend", value_enum, default_value_t = Backend::Singlepass)]
+    pub compiler_backend: Backend,
+
+    /// Whether or not to run packages during tests
+    #[arg(long = "run")]
+    pub run_packages: bool,
+
+    /// The output directory
+    #[arg(short = 'o', long, default_value = get_default_out_path().into_os_string())]
+    pub outdir: std::path::PathBuf,
+
+    /// The authorization token needed to see packages.
+    #[arg(long, default_value_t = get_default_token())]
+    pub auth_token: String,
+
+    /// The number of concurrent tests (jobs) to perform
+    #[arg(long, default_value = <std::num::NonZeroUsize as Into<usize>>::into(std::thread::available_parallelism().unwrap_or(std::num::NonZeroUsize::new(2).unwrap())).to_string()) ]
+    pub jobs: usize,
+}
+
+impl ArgusConfig {
+    pub fn is_compatible(&self, other: &Self) -> bool {
+        self.run_packages == other.run_packages
+    }
+}
diff --git a/tests/wasmer-argus/src/argus/mod.rs b/tests/wasmer-argus/src/argus/mod.rs
new file mode 100644
index 00000000000..59bf6bef020
--- /dev/null
+++ b/tests/wasmer-argus/src/argus/mod.rs
@@ -0,0 +1,302 @@
+mod config;
+mod packages;
+mod result;
+
+pub use config::*;
+
+use indicatif::{MultiProgress, ProgressBar};
+use std::{
+    fs::{File, OpenOptions},
+    io::{BufReader, Write as _},
+    path::PathBuf,
+    sync::Arc,
+    time::Duration,
+};
+use tokio::{
+    sync::{mpsc, Semaphore},
+    task::JoinSet,
+    time,
+};
+use tracing::*;
+use url::Url;
+use wasmer_api::{types::PackageVersionWithPackage, WasmerClient};
+use webc::{
+    v1::{ParseOptions, WebCOwned},
+    v2::read::OwnedReader,
+    Container, Version,
+};
+
+use crate::argus::result::TestResults;
+
+use self::result::TestReport;
+
+#[derive(Debug, Clone)]
+pub struct Argus {
+    pub config: ArgusConfig,
+    pub client: WasmerClient,
+}
+
+impl TryFrom<ArgusConfig> for Argus {
+    type Error = anyhow::Error;
+
+    fn try_from(config: ArgusConfig) -> Result<Self, Self::Error> {
+        let client = WasmerClient::new(Url::parse(&config.registry_url)?, "wasmer-argus")?;
+
+        let client = client.with_auth_token(config.auth_token.clone());
+        Ok(Argus { client, config })
+    }
+}
+
+impl Argus {
+    /// Start the testsuite using the configuration in [`Self::config`]
+    pub async fn run(self) -> anyhow::Result<()> {
+        info!("fetching packages from {}", self.config.registry_url);
+
+        let m = MultiProgress::new();
+        let (s, mut r) = mpsc::unbounded_channel();
+
+        let mut pool = JoinSet::new();
+
+        {
+            let this = self.clone();
+            let bar = m.add(ProgressBar::new(0));
+
+            pool.spawn(async move { this.fetch_packages(s, bar).await });
+        }
+
+        let c = Arc::new(self.config.clone());
+
+        let mut count = 0;
+
+        let sem = Arc::new(Semaphore::new(self.config.jobs));
+
+        while let Some(pkg) = r.recv().await {
+            let c = c.clone();
+            let bar = m.add(ProgressBar::new(0));
+            let permit = Arc::clone(&sem).acquire_owned().await;
+
+            pool.spawn(async move {
+                let _permit = permit;
+                Argus::test(count, c, &pkg, bar).await
+            });
+
+            count = count + 1;
+        }
+
+        while let Some(t) = pool.join_next().await {
+            if let Err(e) = t {
+                error!("task failed: {e}")
+            }
+        }
+
+        info!("done!");
+        Ok(())
+    }
+
+    /// The actual test
+    async fn test<'a>(
+        test_id: u64,
+        config: Arc<ArgusConfig>,
+        pkg: &'a PackageVersionWithPackage,
+        p: ProgressBar,
+    ) -> anyhow::Result<()> {
+        p.set_style(
+            indicatif::ProgressStyle::with_template(&format!(
+                "[{test_id}] {{spinner:.blue}} {{msg}}"
+            ))
+            .unwrap()
+            .tick_strings(&["✶", "✸", "✹", "✺", "✹", "✷"]),
+        );
+
+        p.enable_steady_tick(Duration::from_millis(100));
+
+        let name = Argus::get_package_id(pkg);
+
+        let webc_url: Url = match &pkg.distribution.pirita_download_url {
+            Some(url) => url.parse().unwrap(),
+            None => {
+                info!("package {} has no download url, skipping", name);
+                p.finish_and_clear();
+                return Ok(());
+            }
+        };
+
+        p.set_message(format!("[{test_id}] testing package {name}",));
+
+        let path = Argus::get_path(config.clone(), pkg).await;
+        p.set_message(format!(
+            "testing package {name} -- path to download to is: {:?}",
+            path
+        ));
+
+        Argus::download_package(test_id, &path, &webc_url, &p).await?;
+
+        info!("package downloaded!");
+
+        p.reset();
+        p.set_style(
+            indicatif::ProgressStyle::with_template(&format!(
+                "[{test_id}] {{spinner:.blue}} {{msg}}"
+            ))
+            .unwrap()
+            .tick_strings(&["✶", "✸", "✹", "✺", "✹", "✷"]),
+        );
+
+        p.enable_steady_tick(Duration::from_millis(100));
+
+        p.set_message("package downloaded");
+        let filepath = path.join("package.webc");
+
+        let start = time::Instant::now();
+
+        let res = match std::panic::catch_unwind(|| {
+            p.set_message("reading webc bytes from filesystem");
+            let bytes = std::fs::read(&filepath)?;
+            let store = wasmer::Store::new(config.compiler_backend.to_engine());
+
+            let webc = match webc::detect(bytes.as_slice()) {
+                Ok(Version::V1) => {
+                    let options = ParseOptions::default();
+                    let webc = WebCOwned::parse(bytes, &options)?;
+                    Container::from(webc)
+                }
+                Ok(Version::V2) => Container::from(OwnedReader::parse(bytes)?),
+                Ok(other) => anyhow::bail!("Unsupported version, {other}"),
+                Err(e) => anyhow::bail!("An error occurred: {e}"),
+            };
+
+            p.set_message("created webc");
+
+            for atom in webc.atoms().iter() {
+                info!(
+                    "creating module for atom {} with length {}",
+                    atom.0,
+                    atom.1.len()
+                );
+                p.set_message(format!(
+                    "-- {name} -- creating module for atom {} (has length {} bytes)",
+                    atom.0,
+                    atom.1.len()
+                ));
+                wasmer::Module::new(&store, atom.1.as_slice())?;
+            }
+
+            Ok(())
+        }) {
+            Ok(r) => match r {
+                Ok(_) => Ok(()),
+                Err(e) => Err(format!("{e}")),
+            },
+            Err(e) => Err(format!("{:?}", e)),
+        };
+
+        let time = start - time::Instant::now();
+
+        let report = TestReport::new(config.as_ref(), res, time);
+
+        Argus::write_report(&path, report).await?;
+
+        p.finish_with_message(format!("test for package {name} done!"));
+        p.finish_and_clear();
+
+        Ok(())
+    }
+
+    /// Checks whether or not the package should be tested
+    ///
+    /// This is done by checking if it was already tested in a compatible (i.e. same backend)
+    /// previous run by searching for the a directory with the package name in the directory
+    /// [`PackageVersionWithPackage::package`] with the same `pirita_sha256_hash` as in
+    /// [`PackageVersionWithPackage::distribution`] that contains a file that matches the current
+    /// configuration.
+    ///
+    /// For example, given a package such as
+    /// ```
+    /// {
+    ///     "package": {
+    ///         "package_name": "any/mytest",
+    ///         ...
+    ///     },
+    ///     "distribution": {
+    ///         "pirita_sha256_hash":
+    ///             "47945b31a4169e6c82162d29e3f54cbf7cb979c8e84718a86dec1cc0f6c19890"
+    ///     }
+    ///     ...
+    /// }
+    /// ```
+    ///
+    /// this function will check if there is a file with path
+    /// `any_mytest/47945b31a4169e6c82162d29e3f54cbf7cb979c8e84718a86dec1cc0f6c19890.json`
+    /// in `outdir` as prescribed by [`Self::config`]. If the file contains a compatible test run,
+    /// it returns `false`.
+    /// If the output directory does not exists, this function returns `true`.
+    async fn to_test(&self, pkg: &PackageVersionWithPackage) -> bool {
+        let name = Argus::get_package_id(pkg);
+
+        info!("checking if package {name} needs to be tested or not");
+
+        let dir_path = std::path::PathBuf::from(&self.config.outdir);
+
+        if !dir_path.exists() {
+            return true;
+        }
+
+        if pkg.distribution.pirita_sha256_hash.is_none() {
+            info!("skipping test for {name} as it has no hash");
+            return false;
+        }
+
+        let path = Argus::get_path(Arc::new(self.config.clone()), pkg)
+            .await
+            .join("results.json");
+        if !path.exists() {
+            return true;
+        }
+
+        let file = match File::open(path) {
+            Ok(file) => file,
+            Err(e) => {
+                info!(
+                    "re-running test for pkg {:?} as previous-run file failed to open: {e}",
+                    pkg
+                );
+                return true;
+            }
+        };
+
+        let reader = BufReader::new(file);
+        let prev_run: TestResults = match serde_json::from_reader(reader) {
+            Ok(p) => p,
+            Err(e) => {
+                info!(
+                    "re-running test for pkg {:?} as previous-run file failed to be deserialized: {e}",
+                    pkg
+                );
+                return true;
+            }
+        };
+
+        !prev_run.has(&self.config)
+    }
+
+    async fn write_report(path: &PathBuf, report: TestReport) -> anyhow::Result<()> {
+        let test_results_path = path.join("results.json");
+
+        let mut test_results = if test_results_path.exists() {
+            let s = tokio::fs::read_to_string(&test_results_path).await?;
+            serde_json::from_str(&s)?
+        } else {
+            TestResults::default()
+        };
+
+        test_results.add(report);
+
+        let mut file = OpenOptions::new()
+            .write(true)
+            .create(true)
+            .truncate(true)
+            .open(&test_results_path)?;
+        file.write_all(serde_json::to_string(&test_results).unwrap().as_bytes())?;
+        Ok(())
+    }
+}
diff --git a/tests/wasmer-argus/src/argus/packages.rs b/tests/wasmer-argus/src/argus/packages.rs
new file mode 100644
index 00000000000..7c77396719f
--- /dev/null
+++ b/tests/wasmer-argus/src/argus/packages.rs
@@ -0,0 +1,222 @@
+use super::Argus;
+use crate::ArgusConfig;
+use futures::StreamExt;
+use indicatif::{ProgressBar, ProgressStyle};
+use reqwest::{header, Client};
+use std::{fs::File, io::Write, path::PathBuf, sync::Arc, time::Duration};
+use tokio::sync::mpsc::UnboundedSender;
+use tracing::*;
+use url::Url;
+use wasmer_api::{
+    query::get_package_versions_stream,
+    types::{AllPackageVersionsVars, PackageVersionSortBy, PackageVersionWithPackage},
+};
+
+impl Argus {
+    /// Fetch all packages from the registry
+    #[tracing::instrument(skip(self, s, p))]
+    pub async fn fetch_packages(
+        &self,
+        s: UnboundedSender<PackageVersionWithPackage>,
+        p: ProgressBar,
+    ) -> anyhow::Result<()> {
+        info!("starting to fetch packages..");
+        let vars = AllPackageVersionsVars {
+            sort_by: Some(PackageVersionSortBy::Oldest),
+            ..Default::default()
+        };
+
+        p.set_style(
+            ProgressStyle::with_template("{spinner:.blue} {msg}")
+                .unwrap()
+                .tick_strings(&["✶", "✸", "✹", "✺", "✹", "✷"]),
+        );
+        p.enable_steady_tick(Duration::from_millis(1000));
+
+        let mut count = 0;
+
+        let call = get_package_versions_stream(&self.client, vars.clone());
+        futures::pin_mut!(call);
+        p.set_message("starting to fetch packages..".to_string());
+
+        while let Some(pkgs) = call.next().await {
+            let pkgs = match pkgs {
+                Ok(pkgs) => pkgs,
+                Err(e) => {
+                    error!("failed to fetch packages: {e}");
+                    p.finish_and_clear();
+                    anyhow::bail!("failed to fetch packages: {e}")
+                }
+            };
+            p.set_message(format!("fetched {} packages", count));
+            count = count + pkgs.len();
+
+            for pkg in pkgs {
+                if self.to_test(&pkg).await {
+                    if let Err(e) = s.send(pkg) {
+                        error!("failed to send packages: {e}");
+                        p.finish_and_clear();
+                        anyhow::bail!("failed to send packages: {e}")
+                    };
+                }
+            }
+        }
+
+        p.finish_with_message(format!("fetched {count} packages"));
+        info!("finished fetching packages: fetched {count} packages, closing channel");
+        drop(s);
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(p))]
+    pub(crate) async fn download_package<'a>(
+        test_id: u64,
+        path: &'a PathBuf,
+        url: &'a Url,
+        p: &'a ProgressBar,
+    ) -> anyhow::Result<()> {
+        info!("downloading package from {} to file {:?}", url, path);
+        static APP_USER_AGENT: &str =
+            concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"),);
+
+        if !path.exists() {
+            tokio::fs::create_dir_all(path).await?;
+        } else {
+            if path.exists() && !path.is_dir() {
+                anyhow::bail!("path {:?} exists, but it is not a directory!", path)
+            }
+        }
+
+        let client = Client::builder().user_agent(APP_USER_AGENT).build()?;
+
+        let download_size = {
+            let resp = client.head(url.as_str()).send().await?;
+            if resp.status().is_success() {
+                resp.headers()
+                    .get(header::CONTENT_LENGTH)
+                    .and_then(|ct_len| ct_len.to_str().ok())
+                    .and_then(|ct_len| ct_len.parse().ok())
+                    .unwrap_or(0) // Fallback to 0
+            } else {
+                anyhow::bail!(
+                    "Couldn't fetch head from URL {}. Error: {:?}",
+                    url,
+                    resp.status()
+                )
+            }
+        };
+
+        let request = client.get(url.as_str());
+
+        p.set_length(download_size);
+
+        p.set_style(
+            ProgressStyle::default_bar()
+                .template(&format!(
+                    "[{test_id}] [{{bar:40.cyan/blue}}] {{bytes}}/{{total_bytes}} - {{msg}}"
+                ))
+                .unwrap()
+                .progress_chars("#>-"),
+        );
+
+        p.set_message(format!("downloading from {url}"));
+
+        let mut outfile = match File::create(&path.join("package.webc")) {
+            Ok(o) => o,
+            Err(e) => {
+                error!(
+                    "[{test_id}] failed to create file at {:?}. Error: {e}",
+                    path.join("package.webc")
+                );
+
+                p.finish_and_clear();
+
+                anyhow::bail!(
+                    "[{test_id}] failed to create file at {:?}. Error: {e}",
+                    path.join("package.webc")
+                );
+            }
+        };
+        let mut download = match request.send().await {
+            Ok(d) => d,
+            Err(e) => {
+                error!("[{test_id}] failed to download from URL {url}. Error: {e}");
+                p.finish_and_clear();
+                anyhow::bail!("[{test_id}] failed to download from URL {url}. Error: {e}");
+            }
+        };
+
+        loop {
+            match download.chunk().await {
+                Err(e) => {
+                    error!(
+                        "[{test_id}] failed to download chunk from {:?}. Error: {e}",
+                        download
+                    );
+                    p.finish_and_clear();
+                    anyhow::bail!(
+                        "[{test_id}] failed to download chunk from {:?}. Error: {e}",
+                        download
+                    );
+                }
+                Ok(chunk) => {
+                    if let Some(chunk) = chunk {
+                        p.inc(chunk.len() as u64);
+                        if let Err(e) = outfile.write(&chunk) {
+                            error!(
+                                "[{test_id}] failed to write chunk to file {:?}. Error: {e}",
+                                outfile
+                            );
+                            p.finish_and_clear();
+                            anyhow::bail!(
+                                "[{test_id}] failed to write chunk to file {:?}. Error: {e}",
+                                outfile
+                            );
+                        };
+                    } else {
+                        break;
+                    }
+                }
+            }
+        }
+
+        outfile.flush()?;
+        drop(outfile);
+
+        Ok(())
+    }
+
+    /// Return the complete path to the folder of the test for the package, from the outdir to the
+    /// hash
+    pub async fn get_path<'a>(
+        config: Arc<ArgusConfig>,
+        pkg: &'a PackageVersionWithPackage,
+    ) -> PathBuf {
+        let hash = match &pkg.distribution.pirita_sha256_hash {
+            Some(hash) => hash,
+            None => {
+                unreachable!("no package without an hash should reach this function!")
+            }
+        };
+
+        let _namespace = match &pkg.package.namespace {
+            Some(ns) => ns.replace('/', "_"),
+            None => "unknown_namespace".to_owned(),
+        };
+
+        config.outdir.join(hash)
+    }
+
+    pub fn get_package_id(pkg: &PackageVersionWithPackage) -> String {
+        let namespace = match &pkg.package.namespace {
+            Some(namespace) => namespace.replace('/', "_"),
+            None => String::from("unknown_namespace"),
+        };
+        format!(
+            "{}/{}_v{}",
+            namespace,
+            pkg.package.package_name.replace("/", "_"),
+            pkg.version
+        )
+    }
+}
diff --git a/tests/wasmer-argus/src/argus/result.rs b/tests/wasmer-argus/src/argus/result.rs
new file mode 100644
index 00000000000..b348fe9f7e4
--- /dev/null
+++ b/tests/wasmer-argus/src/argus/result.rs
@@ -0,0 +1,51 @@
+use crate::ArgusConfig;
+use serde::{Deserialize, Serialize};
+use std::{collections::HashMap, time::Duration};
+
+/// The result of a test run
+// [todo] This must support multiple test runs, so fields shall be serializable collections
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct TestResults {
+    // In order to avoid having complex mechanisms to serialize and deserialize
+    // hashmaps with struct keys we use Engines' identifiers as keys
+    results: HashMap<String, TestReport>,
+}
+
+impl TestResults {
+    pub(crate) fn has(&self, config: &ArgusConfig) -> bool {
+        match self.results.get(&config.compiler_backend.to_string()) {
+            Some(prev_result) => {
+                // Ideally we should test more differences between runs,
+                // once this is the case this check can be moved to a function
+                // in ArgusConfig::is_compatible(&self, other: Self) -> bool
+                prev_result.config.is_compatible(config)
+            }
+            None => false,
+        }
+    }
+
+    pub(crate) fn add(&mut self, report: TestReport) {
+        self.results
+            .insert(report.config.compiler_backend.to_string(), report);
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TestReport {
+    /// The outcome of the test
+    result: Result<(), String>,
+    /// How long the test took
+    time: Duration,
+    /// The configuration of the test
+    config: ArgusConfig,
+}
+
+impl TestReport {
+    pub fn new(config: &ArgusConfig, result: Result<(), String>, time: Duration) -> Self {
+        Self {
+            result,
+            time,
+            config: config.clone(),
+        }
+    }
+}
diff --git a/tests/wasmer-argus/src/lib.rs b/tests/wasmer-argus/src/lib.rs
new file mode 100644
index 00000000000..9fe6470319e
--- /dev/null
+++ b/tests/wasmer-argus/src/lib.rs
@@ -0,0 +1,4 @@
+mod argus;
+
+// Simply public re-export of the core struct
+pub use argus::*;
diff --git a/tests/wasmer-argus/src/main.rs b/tests/wasmer-argus/src/main.rs
new file mode 100644
index 00000000000..83a5df0d78a
--- /dev/null
+++ b/tests/wasmer-argus/src/main.rs
@@ -0,0 +1,15 @@
+mod argus;
+
+use argus::*;
+use clap::Parser;
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    tracing_subscriber::fmt::fmt()
+        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+        .init();
+    let config = ArgusConfig::parse();
+
+    let argus = Argus::try_from(config)?;
+    argus.run().await
+}

From 4bd3979153d7a084641f37fc9f29237e4297db44 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 6 Mar 2024 18:06:41 +0100
Subject: [PATCH 02/33] add README.md

---
 tests/wasmer-argus/README.md | 42 ++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 tests/wasmer-argus/README.md

diff --git a/tests/wasmer-argus/README.md b/tests/wasmer-argus/README.md
new file mode 100644
index 00000000000..9809c102e1d
--- /dev/null
+++ b/tests/wasmer-argus/README.md
@@ -0,0 +1,42 @@
+# wasmer-argus
+
+Automatically test packages from the registry. 
+
+## Building
+Simply build with `cargo build --package wasmer-argus`. The `wasmer-argus`
+binary will be in the `target/debug` directory.
+
+## Usage
+This binary fetches packages from the graphql endpoint  of a registry. By
+default, it uses `http://registry.wasmer.io/graphql`; and the needed
+authorization token is retrieved from the environment using the `WASMER_TOKEN`. 
+Users can specify the token via CLI with the appropriate flag.
+
+This testsuite is parallelised, and the degree of parallelism available can be
+specified both by CLI flag or automatically using
+`std::thread::available_parallelism`.
+
+```
+Fetch and test packages from a WebContainer registry
+
+Usage: wasmer-argus [OPTIONS]
+
+Options:
+  -r, --registry-url <REGISTRY_URL>  
+        The GraphQL endpoint of the registry to test [default: http://registry.wasmer.io/graphql]
+  -b, --backend <COMPILER_BACKEND>   
+        The backend to test the compilation against [default: singlepass] [possible values: llvm, singlepass, cranelift]
+      --run                          
+        Whether or not to run packages during tests
+  -o, --outdir <OUTDIR>              
+        The output directory [default: /home/ecmm/sw/wasmer/wasmer/target/debug/out]
+      --auth-token <AUTH_TOKEN>      
+        The authorization token needed to see packages [default: <env::WASMER_TOKEN>]
+      --jobs <JOBS>                  
+        The number of concurrent tests (jobs) to perform [default: 12]
+  -h, --help                         Print help
+  -V, --version                      Print version
+```
+
+
+

From f4da13f4f2ec65fe82daae00ce988af5a851d4b3 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 6 Mar 2024 18:29:28 +0100
Subject: [PATCH 03/33] feat: add llvm to backends

---
 tests/wasmer-argus/Cargo.toml          | 2 +-
 tests/wasmer-argus/src/argus/config.rs | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/tests/wasmer-argus/Cargo.toml b/tests/wasmer-argus/Cargo.toml
index f6d1feadff6..a55f6f4c63e 100644
--- a/tests/wasmer-argus/Cargo.toml
+++ b/tests/wasmer-argus/Cargo.toml
@@ -31,7 +31,7 @@ clap = {version = "4.4.11", features = ["derive", "string"]}
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 serde = { version = "1.0.197", features = ["derive"] }
 serde_json = "1.0.114"
-wasmer = { version = "4.2.6", path = "../../lib/api", features = ["engine", "core", "singlepass", "cranelift"] }
+wasmer = { version = "4.2.6", path = "../../lib/api", features = ["engine", "core", "singlepass", "cranelift", "llvm"] }
 reqwest = "0.11.24"
 derive_more = "0.99.17"
 webc.workspace = true
diff --git a/tests/wasmer-argus/src/argus/config.rs b/tests/wasmer-argus/src/argus/config.rs
index c72fcdf516f..fdde9c18b34 100644
--- a/tests/wasmer-argus/src/argus/config.rs
+++ b/tests/wasmer-argus/src/argus/config.rs
@@ -10,7 +10,7 @@ fn get_default_out_path() -> PathBuf {
 }
 
 fn get_default_token() -> String {
-    std::env::var("WASMER_TOKEN").unwrap_or(String::new())
+    std::env::var("WASMER_TOKEN").unwrap_or_default()
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, ValueEnum, derive_more::Display)]
@@ -23,7 +23,11 @@ pub enum Backend {
 impl Backend {
     pub fn to_engine(&self) -> Engine {
         match self {
-            Backend::LLVM => todo!(),
+            Backend::LLVM => Engine::new(
+                Box::new(wasmer::LLVM::new()),
+                Target::default(),
+                Features::default(),
+            ),
             Backend::Singlepass => Engine::new(
                 Box::new(wasmer::Singlepass::new()),
                 Target::default(),

From 8b029c5d0e783e3fc16833a8d0ea0268d6ad2527 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 6 Mar 2024 18:29:35 +0100
Subject: [PATCH 04/33] fix: clippy warnings

---
 tests/wasmer-argus/src/argus/mod.rs      | 10 +++++-----
 tests/wasmer-argus/src/argus/packages.rs | 12 +++++-------
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/tests/wasmer-argus/src/argus/mod.rs b/tests/wasmer-argus/src/argus/mod.rs
index 59bf6bef020..7c490820833 100644
--- a/tests/wasmer-argus/src/argus/mod.rs
+++ b/tests/wasmer-argus/src/argus/mod.rs
@@ -8,7 +8,7 @@ use indicatif::{MultiProgress, ProgressBar};
 use std::{
     fs::{File, OpenOptions},
     io::{BufReader, Write as _},
-    path::PathBuf,
+    path::Path,
     sync::Arc,
     time::Duration,
 };
@@ -80,7 +80,7 @@ impl Argus {
                 Argus::test(count, c, &pkg, bar).await
             });
 
-            count = count + 1;
+            count += 1;
         }
 
         while let Some(t) = pool.join_next().await {
@@ -94,10 +94,10 @@ impl Argus {
     }
 
     /// The actual test
-    async fn test<'a>(
+    async fn test(
         test_id: u64,
         config: Arc<ArgusConfig>,
-        pkg: &'a PackageVersionWithPackage,
+        pkg: &PackageVersionWithPackage,
         p: ProgressBar,
     ) -> anyhow::Result<()> {
         p.set_style(
@@ -279,7 +279,7 @@ impl Argus {
         !prev_run.has(&self.config)
     }
 
-    async fn write_report(path: &PathBuf, report: TestReport) -> anyhow::Result<()> {
+    async fn write_report(path: &Path, report: TestReport) -> anyhow::Result<()> {
         let test_results_path = path.join("results.json");
 
         let mut test_results = if test_results_path.exists() {
diff --git a/tests/wasmer-argus/src/argus/packages.rs b/tests/wasmer-argus/src/argus/packages.rs
index 7c77396719f..4909c377155 100644
--- a/tests/wasmer-argus/src/argus/packages.rs
+++ b/tests/wasmer-argus/src/argus/packages.rs
@@ -49,7 +49,7 @@ impl Argus {
                 }
             };
             p.set_message(format!("fetched {} packages", count));
-            count = count + pkgs.len();
+            count += pkgs.len();
 
             for pkg in pkgs {
                 if self.to_test(&pkg).await {
@@ -81,10 +81,8 @@ impl Argus {
 
         if !path.exists() {
             tokio::fs::create_dir_all(path).await?;
-        } else {
-            if path.exists() && !path.is_dir() {
+        } else if path.exists() && !path.is_dir() {
                 anyhow::bail!("path {:?} exists, but it is not a directory!", path)
-            }
         }
 
         let client = Client::builder().user_agent(APP_USER_AGENT).build()?;
@@ -188,9 +186,9 @@ impl Argus {
 
     /// Return the complete path to the folder of the test for the package, from the outdir to the
     /// hash
-    pub async fn get_path<'a>(
+    pub async fn get_path(
         config: Arc<ArgusConfig>,
-        pkg: &'a PackageVersionWithPackage,
+        pkg: &PackageVersionWithPackage,
     ) -> PathBuf {
         let hash = match &pkg.distribution.pirita_sha256_hash {
             Some(hash) => hash,
@@ -215,7 +213,7 @@ impl Argus {
         format!(
             "{}/{}_v{}",
             namespace,
-            pkg.package.package_name.replace("/", "_"),
+            pkg.package.package_name.replace('/', "_"),
             pkg.version
         )
     }

From 9f26b825b9819fe16d75bf19591a2b461e62a6b3 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 6 Mar 2024 18:33:28 +0100
Subject: [PATCH 05/33] fix: formatting

---
 tests/wasmer-argus/src/argus/packages.rs | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/tests/wasmer-argus/src/argus/packages.rs b/tests/wasmer-argus/src/argus/packages.rs
index 4909c377155..7662bc4e2fb 100644
--- a/tests/wasmer-argus/src/argus/packages.rs
+++ b/tests/wasmer-argus/src/argus/packages.rs
@@ -82,7 +82,7 @@ impl Argus {
         if !path.exists() {
             tokio::fs::create_dir_all(path).await?;
         } else if path.exists() && !path.is_dir() {
-                anyhow::bail!("path {:?} exists, but it is not a directory!", path)
+            anyhow::bail!("path {:?} exists, but it is not a directory!", path)
         }
 
         let client = Client::builder().user_agent(APP_USER_AGENT).build()?;
@@ -186,10 +186,7 @@ impl Argus {
 
     /// Return the complete path to the folder of the test for the package, from the outdir to the
     /// hash
-    pub async fn get_path(
-        config: Arc<ArgusConfig>,
-        pkg: &PackageVersionWithPackage,
-    ) -> PathBuf {
+    pub async fn get_path(config: Arc<ArgusConfig>, pkg: &PackageVersionWithPackage) -> PathBuf {
         let hash = match &pkg.distribution.pirita_sha256_hash {
             Some(hash) => hash,
             None => {

From f9947001999c2fe8547f1373b7432c911dc6eb63 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 6 Mar 2024 18:51:43 +0100
Subject: [PATCH 06/33] fix: clippy

---
 tests/wasmer-argus/src/argus/config.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/wasmer-argus/src/argus/config.rs b/tests/wasmer-argus/src/argus/config.rs
index fdde9c18b34..c32ab9db42e 100644
--- a/tests/wasmer-argus/src/argus/config.rs
+++ b/tests/wasmer-argus/src/argus/config.rs
@@ -15,7 +15,7 @@ fn get_default_token() -> String {
 
 #[derive(Debug, Clone, Serialize, Deserialize, ValueEnum, derive_more::Display)]
 pub enum Backend {
-    LLVM,
+    Llvm,
     Singlepass,
     Cranelift,
 }
@@ -23,7 +23,7 @@ pub enum Backend {
 impl Backend {
     pub fn to_engine(&self) -> Engine {
         match self {
-            Backend::LLVM => Engine::new(
+            Backend::Llvm => Engine::new(
                 Box::new(wasmer::LLVM::new()),
                 Target::default(),
                 Features::default(),

From 619583065ca574c1ce0e958ffcb9c990af90eac6 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Thu, 7 Mar 2024 09:42:37 +0100
Subject: [PATCH 07/33] fix: clearer code

---
 tests/wasmer-argus/src/argus/config.rs | 8 +++++++-
 tests/wasmer-argus/src/argus/mod.rs    | 6 ++++--
 tests/wasmer-argus/src/main.rs         | 5 ++---
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/tests/wasmer-argus/src/argus/config.rs b/tests/wasmer-argus/src/argus/config.rs
index c32ab9db42e..b694423fc30 100644
--- a/tests/wasmer-argus/src/argus/config.rs
+++ b/tests/wasmer-argus/src/argus/config.rs
@@ -13,6 +13,12 @@ fn get_default_token() -> String {
     std::env::var("WASMER_TOKEN").unwrap_or_default()
 }
 
+fn get_default_jobs() -> usize {
+    std::thread::available_parallelism()
+        .unwrap_or(std::num::NonZeroUsize::new(2).unwrap())
+        .into()
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, ValueEnum, derive_more::Display)]
 pub enum Backend {
     Llvm,
@@ -71,7 +77,7 @@ pub struct ArgusConfig {
     pub auth_token: String,
 
     /// The number of concurrent tests (jobs) to perform
-    #[arg(long, default_value = <std::num::NonZeroUsize as Into<usize>>::into(std::thread::available_parallelism().unwrap_or(std::num::NonZeroUsize::new(2).unwrap())).to_string()) ]
+    #[arg(long, default_value_t = get_default_jobs()) ]
     pub jobs: usize,
 }
 
diff --git a/tests/wasmer-argus/src/argus/mod.rs b/tests/wasmer-argus/src/argus/mod.rs
index 7c490820833..82641428413 100644
--- a/tests/wasmer-argus/src/argus/mod.rs
+++ b/tests/wasmer-argus/src/argus/mod.rs
@@ -149,7 +149,7 @@ impl Argus {
 
         let start = time::Instant::now();
 
-        let res = match std::panic::catch_unwind(|| {
+        let test_exec_result = std::panic::catch_unwind(|| {
             p.set_message("reading webc bytes from filesystem");
             let bytes = std::fs::read(&filepath)?;
             let store = wasmer::Store::new(config.compiler_backend.to_engine());
@@ -182,7 +182,9 @@ impl Argus {
             }
 
             Ok(())
-        }) {
+        });
+
+        let res = match test_exec_result {
             Ok(r) => match r {
                 Ok(_) => Ok(()),
                 Err(e) => Err(format!("{e}")),
diff --git a/tests/wasmer-argus/src/main.rs b/tests/wasmer-argus/src/main.rs
index 83a5df0d78a..327ecf84dea 100644
--- a/tests/wasmer-argus/src/main.rs
+++ b/tests/wasmer-argus/src/main.rs
@@ -5,9 +5,8 @@ use clap::Parser;
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
-    tracing_subscriber::fmt::fmt()
-        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
-        .init();
+    tracing_subscriber::fmt::fmt().init();
+
     let config = ArgusConfig::parse();
 
     let argus = Argus::try_from(config)?;

From 9f066cccd38ac80756f25d4c6d2bab1be2bf6110 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Thu, 7 Mar 2024 09:43:06 +0100
Subject: [PATCH 08/33] fix: use `tokio`'s `File`

---
 tests/wasmer-argus/src/argus/packages.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/wasmer-argus/src/argus/packages.rs b/tests/wasmer-argus/src/argus/packages.rs
index 7662bc4e2fb..f7eff90a568 100644
--- a/tests/wasmer-argus/src/argus/packages.rs
+++ b/tests/wasmer-argus/src/argus/packages.rs
@@ -3,8 +3,8 @@ use crate::ArgusConfig;
 use futures::StreamExt;
 use indicatif::{ProgressBar, ProgressStyle};
 use reqwest::{header, Client};
-use std::{fs::File, io::Write, path::PathBuf, sync::Arc, time::Duration};
-use tokio::sync::mpsc::UnboundedSender;
+use std::{path::PathBuf, sync::Arc, time::Duration};
+use tokio::{fs::File, io::AsyncWriteExt, sync::mpsc::UnboundedSender};
 use tracing::*;
 use url::Url;
 use wasmer_api::{
@@ -119,7 +119,7 @@ impl Argus {
 
         p.set_message(format!("downloading from {url}"));
 
-        let mut outfile = match File::create(&path.join("package.webc")) {
+        let mut outfile = match File::create(&path.join("package.webc")).await {
             Ok(o) => o,
             Err(e) => {
                 error!(
@@ -160,7 +160,7 @@ impl Argus {
                 Ok(chunk) => {
                     if let Some(chunk) = chunk {
                         p.inc(chunk.len() as u64);
-                        if let Err(e) = outfile.write(&chunk) {
+                        if let Err(e) = outfile.write(&chunk).await {
                             error!(
                                 "[{test_id}] failed to write chunk to file {:?}. Error: {e}",
                                 outfile
@@ -178,7 +178,7 @@ impl Argus {
             }
         }
 
-        outfile.flush()?;
+        outfile.flush().await?;
         drop(outfile);
 
         Ok(())

From 53cb9e782d9cb9628c8081c12d07550f4a17fc7a Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Fri, 8 Mar 2024 00:24:07 +0100
Subject: [PATCH 09/33] fix: mark non-rust codeblock as `text`

---
 tests/wasmer-argus/src/argus/mod.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/wasmer-argus/src/argus/mod.rs b/tests/wasmer-argus/src/argus/mod.rs
index 82641428413..6891a3b3a3e 100644
--- a/tests/wasmer-argus/src/argus/mod.rs
+++ b/tests/wasmer-argus/src/argus/mod.rs
@@ -213,7 +213,7 @@ impl Argus {
     /// configuration.
     ///
     /// For example, given a package such as
-    /// ```
+    /// ```text
     /// {
     ///     "package": {
     ///         "package_name": "any/mytest",

From edd40c2e6545859582a25e4702c5da516092f7aa Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Fri, 15 Mar 2024 18:40:47 +0100
Subject: [PATCH 10/33] feat(argus): feature-gate `wasmer_lib`

---
 tests/wasmer-argus/Cargo.toml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tests/wasmer-argus/Cargo.toml b/tests/wasmer-argus/Cargo.toml
index a55f6f4c63e..dd4f8a28c07 100644
--- a/tests/wasmer-argus/Cargo.toml
+++ b/tests/wasmer-argus/Cargo.toml
@@ -15,11 +15,14 @@ path = "src/lib.rs"
 name = "wasmer-argus"
 path = "src/main.rs"
 
+[features]
+wasmer_lib = ["dep:wasmer"]
+
 [dependencies]
 indicatif = "0.17.8"
 # structopt = { version = "0.2", features = [ "paw" ] }
-paw = "1.0"
-wasmer-api = { version = "0.0.23", path = "../../lib/backend-api" }
+# paw = "1.0"
+wasmer-api = { version = "0.0.23", path = "../../lib/backend-api"}
 anyhow = "1.0.80"
 log = "0.4.21"
 cynic = "3.4.3"
@@ -31,7 +34,8 @@ clap = {version = "4.4.11", features = ["derive", "string"]}
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 serde = { version = "1.0.197", features = ["derive"] }
 serde_json = "1.0.114"
-wasmer = { version = "4.2.6", path = "../../lib/api", features = ["engine", "core", "singlepass", "cranelift", "llvm"] }
+wasmer = { version = "4.2.6", path = "../../lib/api", features = ["engine", "core", "singlepass", "cranelift", "llvm"], optional = true }
 reqwest = "0.11.24"
 derive_more = "0.99.17"
 webc.workspace = true
+async-trait = "0.1.77"

From d5b3284ecf8704d8a1db4a7f176a03e797532d3d Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Fri, 15 Mar 2024 18:41:49 +0100
Subject: [PATCH 11/33] feat(argus): add CLI flag for using library

---
 tests/wasmer-argus/src/argus/config.rs | 55 ++++++++++++++------------
 1 file changed, 29 insertions(+), 26 deletions(-)

diff --git a/tests/wasmer-argus/src/argus/config.rs b/tests/wasmer-argus/src/argus/config.rs
index b694423fc30..d9b9ef28da3 100644
--- a/tests/wasmer-argus/src/argus/config.rs
+++ b/tests/wasmer-argus/src/argus/config.rs
@@ -1,7 +1,6 @@
 use clap::ValueEnum;
 use serde::{Deserialize, Serialize};
-use std::path::PathBuf;
-use wasmer::{sys::Features, Engine, NativeEngineExt, Target};
+use std::{path::PathBuf, process::Command};
 
 fn get_default_out_path() -> PathBuf {
     let mut path = std::env::current_dir().unwrap();
@@ -26,28 +25,6 @@ pub enum Backend {
     Cranelift,
 }
 
-impl Backend {
-    pub fn to_engine(&self) -> Engine {
-        match self {
-            Backend::Llvm => Engine::new(
-                Box::new(wasmer::LLVM::new()),
-                Target::default(),
-                Features::default(),
-            ),
-            Backend::Singlepass => Engine::new(
-                Box::new(wasmer::Singlepass::new()),
-                Target::default(),
-                Features::default(),
-            ),
-            Backend::Cranelift => Engine::new(
-                Box::new(wasmer::Cranelift::new()),
-                Target::default(),
-                Features::default(),
-            ),
-        }
-    }
-}
-
 /// Fetch and test packages from a WebContainer registry.
 #[derive(Debug, clap::Parser, Clone, Serialize, Deserialize)]
 #[command(version, about, long_about = None)]
@@ -72,17 +49,43 @@ pub struct ArgusConfig {
     #[arg(short = 'o', long, default_value = get_default_out_path().into_os_string())]
     pub outdir: std::path::PathBuf,
 
-    /// The authorization token needed to see packages.
+    /// The authorization token needed to see packages
     #[arg(long, default_value_t = get_default_token())]
     pub auth_token: String,
 
     /// The number of concurrent tests (jobs) to perform
     #[arg(long, default_value_t = get_default_jobs()) ]
     pub jobs: usize,
+
+    /// The path to the CLI command to use. [default will be searched in $PATH: "wasmer"]
+    #[arg(long)]
+    pub cli_path: Option<String>,
+
+    /// Whether or not this run should use the linked [`wasmer-api`] library instead of the CLI.
+    #[cfg(feature = "wasmer_lib")]
+    #[arg(long, conflicts_with = "cli_path")]
+    pub use_lib: bool,
 }
 
 impl ArgusConfig {
     pub fn is_compatible(&self, other: &Self) -> bool {
-        self.run_packages == other.run_packages
+        self.run_packages == other.run_packages && self.wasmer_version() == other.wasmer_version()
+    }
+
+    pub fn wasmer_version(&self) -> String {
+        #[cfg(feature = "wasmer_lib")]
+        if self.use_lib {
+            return format!("wasmer_lib/{}", env!("CARGO_PKG_VERSION"));
+        }
+
+        let cli_path = match &self.cli_path {
+            Some(p) => p,
+            None => "wasmer",
+        };
+
+        match Command::new(cli_path).arg("--version").output() {
+            Ok(v) => String::from_utf8(v.stdout).unwrap().replace(' ', "/"),
+            Err(e) => panic!("failed to launch cli program {cli_path}: {e}"),
+        }
     }
 }

From f8f21385ef749cc8425178b6114b47cbc5a73899 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Fri, 15 Mar 2024 18:42:34 +0100
Subject: [PATCH 12/33] feat(argus): CLI and library testers

---
 tests/wasmer-argus/src/argus/mod.rs           |  87 +++--------
 .../src/argus/tester/cli_tester.rs            | 139 ++++++++++++++++++
 .../src/argus/tester/lib_tester.rs            |  92 ++++++++++++
 tests/wasmer-argus/src/argus/tester/mod.rs    |  20 +++
 4 files changed, 273 insertions(+), 65 deletions(-)
 create mode 100644 tests/wasmer-argus/src/argus/tester/cli_tester.rs
 create mode 100644 tests/wasmer-argus/src/argus/tester/lib_tester.rs
 create mode 100644 tests/wasmer-argus/src/argus/tester/mod.rs

diff --git a/tests/wasmer-argus/src/argus/mod.rs b/tests/wasmer-argus/src/argus/mod.rs
index 6891a3b3a3e..3240de1b44a 100644
--- a/tests/wasmer-argus/src/argus/mod.rs
+++ b/tests/wasmer-argus/src/argus/mod.rs
@@ -1,9 +1,11 @@
 mod config;
 mod packages;
 mod result;
+mod tester;
 
+use self::result::TestReport;
+use crate::argus::{result::TestResults, tester::Tester};
 pub use config::*;
-
 use indicatif::{MultiProgress, ProgressBar};
 use std::{
     fs::{File, OpenOptions},
@@ -15,20 +17,10 @@ use std::{
 use tokio::{
     sync::{mpsc, Semaphore},
     task::JoinSet,
-    time,
 };
 use tracing::*;
 use url::Url;
 use wasmer_api::{types::PackageVersionWithPackage, WasmerClient};
-use webc::{
-    v1::{ParseOptions, WebCOwned},
-    v2::read::OwnedReader,
-    Container, Version,
-};
-
-use crate::argus::result::TestResults;
-
-use self::result::TestReport;
 
 #[derive(Debug, Clone)]
 pub struct Argus {
@@ -93,7 +85,7 @@ impl Argus {
         Ok(())
     }
 
-    /// The actual test
+    /// Perform the test for a single package
     async fn test(
         test_id: u64,
         config: Arc<ArgusConfig>,
@@ -110,22 +102,22 @@ impl Argus {
 
         p.enable_steady_tick(Duration::from_millis(100));
 
-        let name = Argus::get_package_id(pkg);
+        let package_name = Argus::get_package_id(pkg);
 
         let webc_url: Url = match &pkg.distribution.pirita_download_url {
             Some(url) => url.parse().unwrap(),
             None => {
-                info!("package {} has no download url, skipping", name);
+                info!("package {} has no download url, skipping", package_name);
                 p.finish_and_clear();
                 return Ok(());
             }
         };
 
-        p.set_message(format!("[{test_id}] testing package {name}",));
+        p.set_message(format!("[{test_id}] testing package {package_name}",));
 
         let path = Argus::get_path(config.clone(), pkg).await;
         p.set_message(format!(
-            "testing package {name} -- path to download to is: {:?}",
+            "testing package {package_name} -- path to download to is: {:?}",
             path
         ));
 
@@ -136,7 +128,7 @@ impl Argus {
         p.reset();
         p.set_style(
             indicatif::ProgressStyle::with_template(&format!(
-                "[{test_id}] {{spinner:.blue}} {{msg}}"
+                "[{test_id}/{package_name}] {{spinner:.blue}} {{msg}}"
             ))
             .unwrap()
             .tick_strings(&["✶", "✸", "✹", "✺", "✹", "✷"]),
@@ -145,60 +137,25 @@ impl Argus {
         p.enable_steady_tick(Duration::from_millis(100));
 
         p.set_message("package downloaded");
-        let filepath = path.join("package.webc");
-
-        let start = time::Instant::now();
-
-        let test_exec_result = std::panic::catch_unwind(|| {
-            p.set_message("reading webc bytes from filesystem");
-            let bytes = std::fs::read(&filepath)?;
-            let store = wasmer::Store::new(config.compiler_backend.to_engine());
+        let webc_path = path.join("package.webc");
 
-            let webc = match webc::detect(bytes.as_slice()) {
-                Ok(Version::V1) => {
-                    let options = ParseOptions::default();
-                    let webc = WebCOwned::parse(bytes, &options)?;
-                    Container::from(webc)
-                }
-                Ok(Version::V2) => Container::from(OwnedReader::parse(bytes)?),
-                Ok(other) => anyhow::bail!("Unsupported version, {other}"),
-                Err(e) => anyhow::bail!("An error occurred: {e}"),
-            };
-
-            p.set_message("created webc");
-
-            for atom in webc.atoms().iter() {
-                info!(
-                    "creating module for atom {} with length {}",
-                    atom.0,
-                    atom.1.len()
-                );
-                p.set_message(format!(
-                    "-- {name} -- creating module for atom {} (has length {} bytes)",
-                    atom.0,
-                    atom.1.len()
-                ));
-                wasmer::Module::new(&store, atom.1.as_slice())?;
-            }
-
-            Ok(())
-        });
-
-        let res = match test_exec_result {
-            Ok(r) => match r {
-                Ok(_) => Ok(()),
-                Err(e) => Err(format!("{e}")),
-            },
-            Err(e) => Err(format!("{:?}", e)),
+        #[cfg(feature = "wasmer_lib")]
+        let report = if config.use_lib {
+            tester::lib_tester::LibRunner::run_test(test_id, config, &p, webc_path, &package_name)
+                .await?
+        } else {
+            tester::cli_tester::CLIRunner::run_test(test_id, config, &p, webc_path, &package_name)
+                .await?
         };
 
-        let time = start - time::Instant::now();
-
-        let report = TestReport::new(config.as_ref(), res, time);
+        #[cfg(not(feature = "wasmer_lib"))]
+        let report =
+            tester::cli_tester::CLIRunner::run_test(test_id, config, &p, webc_path, &package_name)
+                .await?;
 
         Argus::write_report(&path, report).await?;
 
-        p.finish_with_message(format!("test for package {name} done!"));
+        p.finish_with_message(format!("test for package {package_name} done!"));
         p.finish_and_clear();
 
         Ok(())
diff --git a/tests/wasmer-argus/src/argus/tester/cli_tester.rs b/tests/wasmer-argus/src/argus/tester/cli_tester.rs
new file mode 100644
index 00000000000..a907be9ecde
--- /dev/null
+++ b/tests/wasmer-argus/src/argus/tester/cli_tester.rs
@@ -0,0 +1,139 @@
+use super::Tester;
+use crate::{argus::result::TestReport, Backend};
+use indicatif::ProgressBar;
+use std::{path::PathBuf, process::Command};
+use tokio::time::{self, Instant};
+use tracing::*;
+use webc::{
+    v1::{ParseOptions, WebCOwned},
+    v2::read::OwnedReader,
+    Container, Version,
+};
+
+#[allow(unused)]
+pub struct CLIRunner {
+    test_id: u64,
+    config: std::sync::Arc<crate::ArgusConfig>,
+    webc_path: PathBuf,
+    package_name: String,
+    start_time: Instant,
+}
+
+impl CLIRunner {
+    pub fn new(
+        test_id: u64,
+        config: std::sync::Arc<crate::ArgusConfig>,
+        webc_path: PathBuf,
+        package_name: &str,
+    ) -> Self {
+        Self {
+            test_id,
+            config,
+            webc_path,
+            package_name: package_name.to_string(),
+            start_time: time::Instant::now(),
+        }
+    }
+
+    pub fn ok(&self) -> anyhow::Result<TestReport> {
+        Ok(TestReport::new(
+            &self.config,
+            Ok(()),
+            time::Instant::now() - self.start_time,
+        ))
+    }
+
+    pub fn err(&self, message: String) -> anyhow::Result<TestReport> {
+        Ok(TestReport::new(
+            &self.config,
+            Err(message),
+            time::Instant::now() - self.start_time,
+        ))
+    }
+
+    async fn test_atom(
+        &self,
+        cli_path: &String,
+        atom: &[u8],
+        dir_path: &PathBuf,
+        atom_id: usize,
+    ) -> anyhow::Result<Result<(), String>> {
+        if let Err(e) = Command::new(cli_path).spawn() {
+            if let std::io::ErrorKind::NotFound = e.kind() {
+                anyhow::bail!("the command '{cli_path}' was not found");
+            }
+        }
+
+        let atom_path = dir_path.join(format!("atom_{atom_id}.wasm"));
+
+        tokio::fs::write(&atom_path, atom).await?;
+
+        let backend = match self.config.compiler_backend {
+            Backend::Llvm => "--llvm",
+            Backend::Singlepass => "--singlepass",
+            Backend::Cranelift => "--cranelift",
+        };
+
+        Ok(
+            match std::panic::catch_unwind(|| {
+                Command::new(cli_path)
+                    .args(["compile", atom_path.to_str().unwrap(), backend])
+                    .output()
+            }) {
+                Ok(r) => match r {
+                    Ok(_) => Ok(()),
+                    Err(e) => Err(e.to_string()),
+                },
+                Err(_) => Err(String::from("thread panicked")),
+            },
+        )
+    }
+}
+
+#[async_trait::async_trait]
+impl Tester for CLIRunner {
+    async fn run_test(
+        test_id: u64,
+        config: std::sync::Arc<crate::ArgusConfig>,
+        p: &ProgressBar,
+        webc_path: PathBuf,
+        package_name: &str,
+    ) -> anyhow::Result<crate::argus::result::TestReport> {
+        let runner = CLIRunner::new(test_id, config, webc_path, package_name);
+
+        let cli_path = match &runner.config.cli_path {
+            Some(ref p) => p.clone(),
+            None => String::from("wasmer"),
+        };
+
+        info!("starting test using CLI at {cli_path}");
+        let mut dir_path = runner.webc_path.clone();
+        dir_path.pop();
+
+        p.set_message(format!("unpacking webc at {:?}", runner.webc_path));
+
+        let bytes = std::fs::read(&runner.webc_path)?;
+
+        let webc = match webc::detect(bytes.as_slice()) {
+            Ok(Version::V1) => {
+                let options = ParseOptions::default();
+                let webc = WebCOwned::parse(bytes, &options)?;
+                Container::from(webc)
+            }
+            Ok(Version::V2) => Container::from(OwnedReader::parse(bytes)?),
+            Ok(other) => return runner.err(format!("Unsupported version, {other}")),
+            Err(e) => return runner.err(format!("An error occurred: {e}")),
+        };
+
+        for (i, atom) in webc.atoms().iter().enumerate() {
+            if let Err(e) = runner
+                .test_atom(&cli_path, atom.1.as_slice(), &dir_path, i)
+                .await?
+            {
+                return runner.err(e);
+            }
+        }
+
+        runner.ok()
+    }
+}
diff --git a/tests/wasmer-argus/src/argus/tester/lib_tester.rs b/tests/wasmer-argus/src/argus/tester/lib_tester.rs
new file mode 100644
index 00000000000..52809e0bf56
--- /dev/null
+++ b/tests/wasmer-argus/src/argus/tester/lib_tester.rs
@@ -0,0 +1,92 @@
+use super::Tester;
+use crate::{Backend, argus::result::TestReport};
+use indicatif::ProgressBar;
+use webc::{Version, v1::{ParseOptions, WebCOwned}, Container, v2::read::OwnedReader};
+use std::path::PathBuf;
+use tokio::time;
+use wasmer::{sys::Features, Engine, NativeEngineExt, Target};
+use tracing::*;
+
+pub struct LibRunner;
+
+#[async_trait::async_trait]
+impl Tester for LibRunner {
+    async fn run_test(
+        test_id: u64,
+        config: std::sync::Arc<crate::ArgusConfig>,
+        p: &ProgressBar,
+        webc_path: PathBuf,
+        package_name: &str,
+    ) -> anyhow::Result<crate::argus::result::TestReport> {
+        let start = time::Instant::now();
+
+        let test_exec_result = std::panic::catch_unwind(|| {
+            p.set_message("reading webc bytes from filesystem");
+            let bytes = std::fs::read(&webc_path)?;
+            let store = wasmer::Store::new(Self::backend_to_engine(&config.compiler_backend));
+
+            let webc = match webc::detect(bytes.as_slice()) {
+                Ok(Version::V1) => {
+                    let options = ParseOptions::default();
+                    let webc = WebCOwned::parse(bytes, &options)?;
+                    Container::from(webc)
+                }
+                Ok(Version::V2) => Container::from(OwnedReader::parse(bytes)?),
+                Ok(other) => anyhow::bail!("Unsupported version, {other}"),
+                Err(e) => anyhow::bail!("An error occurred: {e}"),
+            };
+
+            p.set_message("created webc");
+
+            for atom in webc.atoms().iter() {
+                info!(
+                    "creating module for atom {} with length {}",
+                    atom.0,
+                    atom.1.len()
+                );
+                p.set_message(format!(
+                    "-- {package_name} -- creating module for atom {} (has length {} bytes)",
+                    atom.0,
+                    atom.1.len()
+                ));
+                wasmer::Module::new(&store, atom.1.as_slice())?;
+            }
+
+            Ok(())
+        });
+
+        let res = match test_exec_result {
+            Ok(r) => match r {
+                Ok(_) => Ok(()),
+                Err(e) => Err(format!("{e}")),
+            },
+            Err(e) => Err(format!("{:?}", e)),
+        };
+
+        let time = start - time::Instant::now();
+
+        Ok(TestReport::new(config.as_ref(), res, time))
+    }
+}
+
+impl LibRunner {
+    pub fn backend_to_engine(backend: &Backend) -> Engine {
+        match backend {
+            Backend::Llvm => Engine::new(
+                Box::new(wasmer::LLVM::new()),
+                Target::default(),
+                Features::default(),
+            ),
+            Backend::Singlepass => Engine::new(
+                Box::new(wasmer::Singlepass::new()),
+                Target::default(),
+                Features::default(),
+            ),
+            Backend::Cranelift => Engine::new(
+                Box::new(wasmer::Cranelift::new()),
+                Target::default(),
+                Features::default(),
+            ),
+        }
+    }
+}
diff --git a/tests/wasmer-argus/src/argus/tester/mod.rs b/tests/wasmer-argus/src/argus/tester/mod.rs
new file mode 100644
index 00000000000..746b906adc2
--- /dev/null
+++ b/tests/wasmer-argus/src/argus/tester/mod.rs
@@ -0,0 +1,20 @@
+use super::result::TestReport;
+use crate::ArgusConfig;
+use indicatif::ProgressBar;
+use std::{path::PathBuf, sync::Arc};
+
+pub(crate) mod cli_tester;
+
+#[cfg(feature = "wasmer_lib")]
+pub(crate) mod lib_tester;
+
+#[async_trait::async_trait]
+pub(crate) trait Tester {
+    async fn run_test(
+        test_id: u64,
+        config: Arc<ArgusConfig>,
+        p: &ProgressBar,
+        webc_path: PathBuf,
+        package_name: &str,
+    ) -> anyhow::Result<TestReport>;
+}

From 55f3f7b181689cdedb560700644989f5359cbc31 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Fri, 15 Mar 2024 18:42:49 +0100
Subject: [PATCH 13/33] feat(argus): README with build instructions

---
 tests/wasmer-argus/README.md | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/tests/wasmer-argus/README.md b/tests/wasmer-argus/README.md
index 9809c102e1d..f319986a306 100644
--- a/tests/wasmer-argus/README.md
+++ b/tests/wasmer-argus/README.md
@@ -3,8 +3,21 @@
 Automatically test packages from the registry. 
 
 ## Building
-Simply build with `cargo build --package wasmer-argus`. The `wasmer-argus`
-binary will be in the `target/debug` directory.
+If you want to use the local `wasmer` crate, you shall 
+build the project with `cargo build --package wasmer-argus --features wasmer_lib`.
+
+On macOS, you may encounter an error where the linker does not find `zstd`: a possible 
+solution to this problem is to install `zstd` using `brew` (`brew install zstd`) and 
+using the following command: 
+
+`RUSTFLAGS="-L$(brew --prefix)/lib" cargo build --package wasmer-argus --features wasmer_lib`
+
+Another possiblity is to add the your brew prefix with `/lib` (probably = `/opt/homebrew/lib/`) 
+to the global Cargo config something like:
+```
+[target.aarch64-apple-darwin]
+rustflags = ["-L/opt/homebrew/lib"]
+```
 
 ## Usage
 This binary fetches packages from the graphql endpoint  of a registry. By

From 474f99c0541676981be76dcbe4fd222d6c473919 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Mon, 18 Mar 2024 09:32:24 +0100
Subject: [PATCH 14/33] feat(argus): remove `paw`, add `async_trait`

---
 Cargo.lock | 29 +----------------------------
 1 file changed, 1 insertion(+), 28 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 43efe51baec..a09bcd9781e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3262,33 +3262,6 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd"
 
-[[package]]
-name = "paw"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09c0fc9b564dbc3dc2ed7c92c0c144f4de340aa94514ce2b446065417c4084e9"
-dependencies = [
- "paw-attributes",
- "paw-raw",
-]
-
-[[package]]
-name = "paw-attributes"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f35583365be5d148e959284f42526841917b7bfa09e2d1a7ad5dde2cf0eaa39"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "paw-raw"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f0b59668fe80c5afe998f0c0bf93322bf2cd66cafeeb80581f291716f3467f2"
-
 [[package]]
 name = "pbkdf2"
 version = "0.12.2"
@@ -6146,13 +6119,13 @@ name = "wasmer-argus"
 version = "4.2.6"
 dependencies = [
  "anyhow",
+ "async-trait",
  "clap",
  "cynic",
  "derive_more",
  "futures 0.3.30",
  "indicatif",
  "log",
- "paw",
  "reqwest",
  "serde",
  "serde_json",

From 2793f74420e5ce8346f676ab6dfe91a484ba0f50 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Mon, 18 Mar 2024 09:36:17 +0100
Subject: [PATCH 15/33] fix(argus): fmt

---
 tests/wasmer-argus/src/argus/tester/lib_tester.rs | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tests/wasmer-argus/src/argus/tester/lib_tester.rs b/tests/wasmer-argus/src/argus/tester/lib_tester.rs
index 52809e0bf56..29e91a5c98c 100644
--- a/tests/wasmer-argus/src/argus/tester/lib_tester.rs
+++ b/tests/wasmer-argus/src/argus/tester/lib_tester.rs
@@ -1,11 +1,15 @@
 use super::Tester;
-use crate::{Backend, argus::result::TestReport};
+use crate::{argus::result::TestReport, Backend};
 use indicatif::ProgressBar;
-use webc::{Version, v1::{ParseOptions, WebCOwned}, Container, v2::read::OwnedReader};
 use std::path::PathBuf;
 use tokio::time;
-use wasmer::{sys::Features, Engine, NativeEngineExt, Target};
 use tracing::*;
+use wasmer::{sys::Features, Engine, NativeEngineExt, Target};
+use webc::{
+    v1::{ParseOptions, WebCOwned},
+    v2::read::OwnedReader,
+    Container, Version,
+};
 
 pub struct LibRunner;
 

From 82a45e954c763d87b22d0ee983a9727077f22492 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Mon, 18 Mar 2024 09:49:29 +0100
Subject: [PATCH 16/33] fix(argus): lint

---
 tests/wasmer-argus/src/argus/tester/cli_tester.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/wasmer-argus/src/argus/tester/cli_tester.rs b/tests/wasmer-argus/src/argus/tester/cli_tester.rs
index a907be9ecde..d11007b09ab 100644
--- a/tests/wasmer-argus/src/argus/tester/cli_tester.rs
+++ b/tests/wasmer-argus/src/argus/tester/cli_tester.rs
@@ -1,7 +1,7 @@
 use super::Tester;
 use crate::{argus::result::TestReport, Backend};
 use indicatif::ProgressBar;
-use std::{path::PathBuf, process::Command};
+use std::{path::Path, path::PathBuf, process::Command};
 use tokio::time::{self, Instant};
 use tracing::*;
 use webc::{
@@ -55,7 +55,7 @@ impl CLIRunner {
         &self,
         cli_path: &String,
         atom: &[u8],
-        dir_path: &PathBuf,
+        dir_path: &Path,
         atom_id: usize,
     ) -> anyhow::Result<Result<(), String>> {
         if let Err(e) = Command::new(cli_path).spawn() {

From db10a7015df4a1b17d22e19d3d9cc37c8f523163 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 20 Mar 2024 11:59:36 +0100
Subject: [PATCH 17/33] feat(argus): add output path to compile command

---
 .../src/argus/tester/cli_tester.rs            | 26 +++++++++++++++----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/tests/wasmer-argus/src/argus/tester/cli_tester.rs b/tests/wasmer-argus/src/argus/tester/cli_tester.rs
index d11007b09ab..f140b6e6817 100644
--- a/tests/wasmer-argus/src/argus/tester/cli_tester.rs
+++ b/tests/wasmer-argus/src/argus/tester/cli_tester.rs
@@ -58,13 +58,14 @@ impl CLIRunner {
         dir_path: &Path,
         atom_id: usize,
     ) -> anyhow::Result<Result<(), String>> {
-        if let Err(e) = Command::new(cli_path).spawn() {
+        if let Err(e) = Command::new(cli_path).arg("-V").output() {
             if let std::io::ErrorKind::NotFound = e.kind() {
                 anyhow::bail!("the command '{cli_path}' was not found");
             }
         }
 
         let atom_path = dir_path.join(format!("atom_{atom_id}.wasm"));
+        let output_path = dir_path.join(format!("atom_{atom_id}.wasmu"));
 
         tokio::fs::write(&atom_path, atom).await?;
 
@@ -75,10 +76,24 @@ impl CLIRunner {
         };
 
         Ok(
-            match std::panic::catch_unwind(|| {
-                Command::new(cli_path)
-                    .args(["compile", atom_path.to_str().unwrap(), backend])
-                    .output()
+            match std::panic::catch_unwind(move || {
+                let mut cmd = Command::new(cli_path);
+
+                let cmd = cmd.args([
+                    "compile",
+                    atom_path.to_str().unwrap(),
+                    backend,
+                    "-o",
+                    output_path.to_str().unwrap(),
+                ]);
+
+                info!("running cmd: {:?}", cmd);
+
+                let out = cmd.output();
+
+                info!("run cmd that gave result: {:?}", out);
+
+                out
             }) {
                 Ok(r) => match r {
                     Ok(_) => Ok(()),
@@ -126,6 +141,7 @@ impl Tester for CLIRunner {
         };
 
         for (i, atom) in webc.atoms().iter().enumerate() {
+            p.set_message(format!("testing atom #{i}"));
             if let Err(e) = runner
                 .test_atom(&cli_path, atom.1.as_slice(), &dir_path, i)
                 .await?

From 9673a2266dd8bc54c4477ee0f8f271489e68bda9 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 20 Mar 2024 12:00:29 +0100
Subject: [PATCH 18/33] fix(argus): underscore unused parameter

---
 tests/wasmer-argus/src/argus/tester/lib_tester.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/wasmer-argus/src/argus/tester/lib_tester.rs b/tests/wasmer-argus/src/argus/tester/lib_tester.rs
index 29e91a5c98c..63963e3b15a 100644
--- a/tests/wasmer-argus/src/argus/tester/lib_tester.rs
+++ b/tests/wasmer-argus/src/argus/tester/lib_tester.rs
@@ -16,7 +16,7 @@ pub struct LibRunner;
 #[async_trait::async_trait]
 impl Tester for LibRunner {
     async fn run_test(
-        test_id: u64,
+        _test_id: u64,
         config: std::sync::Arc<crate::ArgusConfig>,
         p: &ProgressBar,
         webc_path: PathBuf,

From ccffb2278ac89731eb58ed58e1759d164b7285c8 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 20 Mar 2024 12:01:16 +0100
Subject: [PATCH 19/33] fix(argus): more logging

---
 tests/wasmer-argus/src/argus/config.rs | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/tests/wasmer-argus/src/argus/config.rs b/tests/wasmer-argus/src/argus/config.rs
index d9b9ef28da3..4ec744c45f1 100644
--- a/tests/wasmer-argus/src/argus/config.rs
+++ b/tests/wasmer-argus/src/argus/config.rs
@@ -1,6 +1,7 @@
 use clap::ValueEnum;
 use serde::{Deserialize, Serialize};
 use std::{path::PathBuf, process::Command};
+use tracing::*;
 
 fn get_default_out_path() -> PathBuf {
     let mut path = std::env::current_dir().unwrap();
@@ -69,7 +70,9 @@ pub struct ArgusConfig {
 
 impl ArgusConfig {
     pub fn is_compatible(&self, other: &Self) -> bool {
-        self.run_packages == other.run_packages && self.wasmer_version() == other.wasmer_version()
+        // Ideally, in the future, we could add more features that could make us conclude that we
+        // need to run tests again.
+        self.run_packages == other.run_packages
     }
 
     pub fn wasmer_version(&self) -> String {
@@ -83,8 +86,21 @@ impl ArgusConfig {
             None => "wasmer",
         };
 
-        match Command::new(cli_path).arg("--version").output() {
-            Ok(v) => String::from_utf8(v.stdout).unwrap().replace(' ', "/"),
+        let mut cmd = Command::new(cli_path);
+        let cmd = cmd.arg("-V");
+
+        info!("running cmd: {:?}", cmd);
+
+        let out = cmd.output();
+
+        info!("run cmd that gave result: {:?}", out);
+
+        match out {
+            Ok(v) => String::from_utf8(v.stdout)
+                .unwrap()
+                .replace(' ', "/")
+                .trim()
+                .to_string(),
             Err(e) => panic!("failed to launch cli program {cli_path}: {e}"),
         }
     }

From fbd7c7397f5dbea026cd4830403f97b1052b80e8 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 20 Mar 2024 12:01:47 +0100
Subject: [PATCH 20/33] fix(argus): reasonable test indexing

---
 tests/wasmer-argus/src/argus/result.rs | 37 +++++++++++++++++---------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/tests/wasmer-argus/src/argus/result.rs b/tests/wasmer-argus/src/argus/result.rs
index b348fe9f7e4..841fd09a207 100644
--- a/tests/wasmer-argus/src/argus/result.rs
+++ b/tests/wasmer-argus/src/argus/result.rs
@@ -2,31 +2,42 @@ use crate::ArgusConfig;
 use serde::{Deserialize, Serialize};
 use std::{collections::HashMap, time::Duration};
 
+type WasmerVersion = String;
+type EngineId = String;
+
 /// The result of a test run
-// [todo] This must support multiple test runs, so fields shall be serializable collections
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct TestResults {
-    // In order to avoid having complex mechanisms to serialize and deserialize
-    // hashmaps with struct keys we use Engines' identifiers as keys
-    results: HashMap<String, TestReport>,
+    results: HashMap<WasmerVersion, HashMap<EngineId, TestReport>>,
 }
 
 impl TestResults {
     pub(crate) fn has(&self, config: &ArgusConfig) -> bool {
-        match self.results.get(&config.compiler_backend.to_string()) {
-            Some(prev_result) => {
-                // Ideally we should test more differences between runs,
-                // once this is the case this check can be moved to a function
-                // in ArgusConfig::is_compatible(&self, other: Self) -> bool
-                prev_result.config.is_compatible(config)
+        let wasmer_version = config.wasmer_version();
+        let engine_id = config.compiler_backend.to_string();
+
+        if let Some(v) = self.results.get(&wasmer_version) {
+            if let Some(v) = v.get(&engine_id) {
+                return v.config.is_compatible(config);
             }
-            None => false,
         }
+
+        false
     }
 
     pub(crate) fn add(&mut self, report: TestReport) {
-        self.results
-            .insert(report.config.compiler_backend.to_string(), report);
+        let wasmer_version = report.config.wasmer_version();
+        let engine_id = report.config.compiler_backend.to_string();
+
+        match self.results.get_mut(&wasmer_version) {
+            Some(v) => _ = v.insert(engine_id, report),
+            None => {
+                _ = self.results.insert(
+                    wasmer_version,
+                    HashMap::from_iter(vec![(engine_id, report)]),
+                )
+            }
+        }
     }
 }
 

From df806d4d824c8e87f9894c3e163fc7354e609c9b Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 20 Mar 2024 12:02:15 +0100
Subject: [PATCH 21/33] fix(argus): explicit env filter

---
 tests/wasmer-argus/src/main.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/wasmer-argus/src/main.rs b/tests/wasmer-argus/src/main.rs
index 327ecf84dea..18a684c52e6 100644
--- a/tests/wasmer-argus/src/main.rs
+++ b/tests/wasmer-argus/src/main.rs
@@ -2,10 +2,14 @@ mod argus;
 
 use argus::*;
 use clap::Parser;
+use tracing_subscriber::{fmt, prelude::*, EnvFilter};
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
-    tracing_subscriber::fmt::fmt().init();
+    tracing_subscriber::registry()
+        .with(fmt::layer())
+        .with(EnvFilter::from_default_env())
+        .init();
 
     let config = ArgusConfig::parse();
 

From e9d53d87fba9b8e71a65e9eecfa8cacdd5710592 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 20 Mar 2024 12:10:54 +0100
Subject: [PATCH 22/33] feat(argus): add package identifier in test
 serialization

---
 tests/wasmer-argus/src/argus/mod.rs    | 6 +++---
 tests/wasmer-argus/src/argus/result.rs | 8 ++++++++
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/tests/wasmer-argus/src/argus/mod.rs b/tests/wasmer-argus/src/argus/mod.rs
index 3240de1b44a..5af484bd035 100644
--- a/tests/wasmer-argus/src/argus/mod.rs
+++ b/tests/wasmer-argus/src/argus/mod.rs
@@ -153,7 +153,7 @@ impl Argus {
             tester::cli_tester::CLIRunner::run_test(test_id, config, &p, webc_path, &package_name)
                 .await?;
 
-        Argus::write_report(&path, report).await?;
+        Argus::write_report(&path, report, package_name.clone()).await?;
 
         p.finish_with_message(format!("test for package {package_name} done!"));
         p.finish_and_clear();
@@ -238,14 +238,14 @@ impl Argus {
         !prev_run.has(&self.config)
     }
 
-    async fn write_report(path: &Path, report: TestReport) -> anyhow::Result<()> {
+    async fn write_report(path: &Path, report: TestReport, pkg_id: String) -> anyhow::Result<()> {
         let test_results_path = path.join("results.json");
 
         let mut test_results = if test_results_path.exists() {
             let s = tokio::fs::read_to_string(&test_results_path).await?;
             serde_json::from_str(&s)?
         } else {
-            TestResults::default()
+            TestResults::from_package_id(pkg_id)
         };
 
         test_results.add(report);
diff --git a/tests/wasmer-argus/src/argus/result.rs b/tests/wasmer-argus/src/argus/result.rs
index 841fd09a207..b061cc2e1dd 100644
--- a/tests/wasmer-argus/src/argus/result.rs
+++ b/tests/wasmer-argus/src/argus/result.rs
@@ -8,6 +8,7 @@ type EngineId = String;
 /// The result of a test run
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct TestResults {
+    package_id: String,
     results: HashMap<WasmerVersion, HashMap<EngineId, TestReport>>,
 }
 
@@ -39,6 +40,13 @@ impl TestResults {
             }
         }
     }
+
+    pub(crate) fn from_package_id(package_id: String) -> Self {
+        Self {
+            package_id,
+            ..Default::default()
+        }
+    }
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

From 055d23085e76b00a55f84ffd38ed8a5a0358f370 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 20 Mar 2024 12:17:27 +0100
Subject: [PATCH 23/33] fix(argus): remove version for crate `wasmer-api`

---
 tests/wasmer-argus/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/wasmer-argus/Cargo.toml b/tests/wasmer-argus/Cargo.toml
index dd4f8a28c07..7729359868c 100644
--- a/tests/wasmer-argus/Cargo.toml
+++ b/tests/wasmer-argus/Cargo.toml
@@ -22,7 +22,7 @@ wasmer_lib = ["dep:wasmer"]
 indicatif = "0.17.8"
 # structopt = { version = "0.2", features = [ "paw" ] }
 # paw = "1.0"
-wasmer-api = { version = "0.0.23", path = "../../lib/backend-api"}
+wasmer-api = { path = "../../lib/backend-api"}
 anyhow = "1.0.80"
 log = "0.4.21"
 cynic = "3.4.3"

From f4f1aa29732d97782123279d28819b5ab222b4d4 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 20 Mar 2024 14:44:15 +0100
Subject: [PATCH 24/33] fix: remove version from backend crate

---
 tests/wasmer-argus/Cargo.toml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/wasmer-argus/Cargo.toml b/tests/wasmer-argus/Cargo.toml
index 7729359868c..8be56bfabcc 100644
--- a/tests/wasmer-argus/Cargo.toml
+++ b/tests/wasmer-argus/Cargo.toml
@@ -20,9 +20,6 @@ wasmer_lib = ["dep:wasmer"]
 
 [dependencies]
 indicatif = "0.17.8"
-# structopt = { version = "0.2", features = [ "paw" ] }
-# paw = "1.0"
-wasmer-api = { path = "../../lib/backend-api"}
 anyhow = "1.0.80"
 log = "0.4.21"
 cynic = "3.4.3"
@@ -39,3 +36,4 @@ reqwest = "0.11.24"
 derive_more = "0.99.17"
 webc.workspace = true
 async-trait = "0.1.77"
+wasmer-api = { path = "../../lib/backend-api" }

From 15bf27aae3bed6573faa599b23ecee9ee5c52bec Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 20 Mar 2024 15:31:09 +0100
Subject: [PATCH 25/33] feat(argus): bump workspace version

---
 Cargo.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index e91eaa08948..c5fd71fb11d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6123,7 +6123,7 @@ dependencies = [
 
 [[package]]
 name = "wasmer-argus"
-version = "4.2.6"
+version = "4.2.7"
 dependencies = [
  "anyhow",
  "async-trait",

From d60c56104085e7bf0e60b364ac6830780063b218 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 20 Mar 2024 15:31:35 +0100
Subject: [PATCH 26/33] refactor(argus): remove lib

---
 tests/wasmer-argus/Cargo.toml | 7 -------
 tests/wasmer-argus/src/lib.rs | 4 ----
 2 files changed, 11 deletions(-)
 delete mode 100644 tests/wasmer-argus/src/lib.rs

diff --git a/tests/wasmer-argus/Cargo.toml b/tests/wasmer-argus/Cargo.toml
index 8be56bfabcc..a0306e10cbb 100644
--- a/tests/wasmer-argus/Cargo.toml
+++ b/tests/wasmer-argus/Cargo.toml
@@ -8,13 +8,6 @@ repository.workspace = true
 rust-version.workspace = true
 version.workspace = true
 
-[lib]
-path = "src/lib.rs"
-
-[[bin]]
-name = "wasmer-argus"
-path = "src/main.rs"
-
 [features]
 wasmer_lib = ["dep:wasmer"]
 
diff --git a/tests/wasmer-argus/src/lib.rs b/tests/wasmer-argus/src/lib.rs
deleted file mode 100644
index 9fe6470319e..00000000000
--- a/tests/wasmer-argus/src/lib.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-mod argus;
-
-// Simply public re-export of the core struct
-pub use argus::*;

From 87f275f0df5770f664f6cdd32c92f1596ebbb61a Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 20 Mar 2024 15:34:34 +0100
Subject: [PATCH 27/33] refactor(argus): remove run flag

BREAKING CHANGE: The `run` flag will no longer be available, as at this
moment there is no utility for it in the general case.

  #
---
 tests/wasmer-argus/src/argus/config.rs | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tests/wasmer-argus/src/argus/config.rs b/tests/wasmer-argus/src/argus/config.rs
index 4ec744c45f1..d32dfb487cf 100644
--- a/tests/wasmer-argus/src/argus/config.rs
+++ b/tests/wasmer-argus/src/argus/config.rs
@@ -42,10 +42,6 @@ pub struct ArgusConfig {
     #[arg(short = 'b', long = "backend", value_enum, default_value_t = Backend::Singlepass)]
     pub compiler_backend: Backend,
 
-    /// Whether or not to run packages during tests
-    #[arg(long = "run")]
-    pub run_packages: bool,
-
     /// The output directory
     #[arg(short = 'o', long, default_value = get_default_out_path().into_os_string())]
     pub outdir: std::path::PathBuf,
@@ -69,10 +65,12 @@ pub struct ArgusConfig {
 }
 
 impl ArgusConfig {
-    pub fn is_compatible(&self, other: &Self) -> bool {
+    pub fn is_compatible(&self, _other: &Self) -> bool {
         // Ideally, in the future, we could add more features that could make us conclude that we
         // need to run tests again.
-        self.run_packages == other.run_packages
+
+        // By default, no config is compatible with the other, that is, we re-run the tests.
+        false
     }
 
     pub fn wasmer_version(&self) -> String {

From e942000876040f144e96e672bc631ac7575fab56 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 20 Mar 2024 20:42:41 +0100
Subject: [PATCH 28/33] refactor(argus): remove `wasmer_version` from
 `ArgusConfig`

---
 tests/wasmer-argus/src/argus/config.rs | 44 ++------------------------
 1 file changed, 2 insertions(+), 42 deletions(-)

diff --git a/tests/wasmer-argus/src/argus/config.rs b/tests/wasmer-argus/src/argus/config.rs
index d32dfb487cf..7651f56bf08 100644
--- a/tests/wasmer-argus/src/argus/config.rs
+++ b/tests/wasmer-argus/src/argus/config.rs
@@ -1,7 +1,6 @@
 use clap::ValueEnum;
 use serde::{Deserialize, Serialize};
-use std::{path::PathBuf, process::Command};
-use tracing::*;
+use std::path::PathBuf;
 
 fn get_default_out_path() -> PathBuf {
     let mut path = std::env::current_dir().unwrap();
@@ -58,48 +57,9 @@ pub struct ArgusConfig {
     #[arg(long)]
     pub cli_path: Option<String>,
 
+
     /// Whether or not this run should use the linked [`wasmer-api`] library instead of the CLI.
     #[cfg(feature = "wasmer_lib")]
     #[arg(long, conflicts_with = "cli_path")]
     pub use_lib: bool,
 }
-
-impl ArgusConfig {
-    pub fn is_compatible(&self, _other: &Self) -> bool {
-        // Ideally, in the future, we could add more features that could make us conclude that we
-        // need to run tests again.
-
-        // By default, no config is compatible with the other, that is, we re-run the tests.
-        false
-    }
-
-    pub fn wasmer_version(&self) -> String {
-        #[cfg(feature = "wasmer_lib")]
-        if self.use_lib {
-            return format!("wasmer_lib/{}", env!("CARGO_PKG_VERSION"));
-        }
-
-        let cli_path = match &self.cli_path {
-            Some(p) => p,
-            None => "wasmer",
-        };
-
-        let mut cmd = Command::new(cli_path);
-        let cmd = cmd.arg("-V");
-
-        info!("running cmd: {:?}", cmd);
-
-        let out = cmd.output();
-
-        info!("run cmd that gave result: {:?}", out);
-
-        match out {
-            Ok(v) => String::from_utf8(v.stdout)
-                .unwrap()
-                .replace(' ', "/")
-                .trim()
-                .to_string(),
-            Err(e) => panic!("failed to launch cli program {cli_path}: {e}"),
-        }
-    }
-}

From 8c6f99f54daa726e638f587013412b48f579a9bf Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 20 Mar 2024 20:43:47 +0100
Subject: [PATCH 29/33] refactor(argus): more autonomy to testers

---
 tests/wasmer-argus/src/argus/mod.rs           | 147 ++++----------
 tests/wasmer-argus/src/argus/packages.rs      |   1 +
 tests/wasmer-argus/src/argus/result.rs        |  70 -------
 .../src/argus/tester/cli_tester.rs            | 189 +++++++++++++-----
 .../src/argus/tester/lib_tester.rs            | 153 ++++++++++----
 tests/wasmer-argus/src/argus/tester/mod.rs    |  71 +++++--
 6 files changed, 351 insertions(+), 280 deletions(-)
 delete mode 100644 tests/wasmer-argus/src/argus/result.rs

diff --git a/tests/wasmer-argus/src/argus/mod.rs b/tests/wasmer-argus/src/argus/mod.rs
index 5af484bd035..82673428a6b 100644
--- a/tests/wasmer-argus/src/argus/mod.rs
+++ b/tests/wasmer-argus/src/argus/mod.rs
@@ -1,19 +1,11 @@
 mod config;
 mod packages;
-mod result;
 mod tester;
 
-use self::result::TestReport;
-use crate::argus::{result::TestResults, tester::Tester};
+use self::tester::{TestReport, Tester};
 pub use config::*;
 use indicatif::{MultiProgress, ProgressBar};
-use std::{
-    fs::{File, OpenOptions},
-    io::{BufReader, Write as _},
-    path::Path,
-    sync::Arc,
-    time::Duration,
-};
+use std::{fs::OpenOptions, io::Write as _, path::Path, sync::Arc, time::Duration};
 use tokio::{
     sync::{mpsc, Semaphore},
     task::JoinSet,
@@ -48,18 +40,18 @@ impl Argus {
         let (s, mut r) = mpsc::unbounded_channel();
 
         let mut pool = JoinSet::new();
+        let c = Arc::new(self.config.clone());
 
         {
             let this = self.clone();
             let bar = m.add(ProgressBar::new(0));
 
-            pool.spawn(async move { this.fetch_packages(s, bar).await });
+            pool.spawn(async move { this.fetch_packages(s, bar, c.clone()).await });
         }
 
-        let c = Arc::new(self.config.clone());
-
         let mut count = 0;
 
+        let c = Arc::new(self.config.clone());
         let sem = Arc::new(Semaphore::new(self.config.jobs));
 
         while let Some(pkg) = r.recv().await {
@@ -89,7 +81,7 @@ impl Argus {
     async fn test(
         test_id: u64,
         config: Arc<ArgusConfig>,
-        pkg: &PackageVersionWithPackage,
+        package: &PackageVersionWithPackage,
         p: ProgressBar,
     ) -> anyhow::Result<()> {
         p.set_style(
@@ -102,9 +94,8 @@ impl Argus {
 
         p.enable_steady_tick(Duration::from_millis(100));
 
-        let package_name = Argus::get_package_id(pkg);
-
-        let webc_url: Url = match &pkg.distribution.pirita_download_url {
+        let package_name = Argus::get_package_id(package);
+        let webc_url: Url = match &package.distribution.pirita_download_url {
             Some(url) => url.parse().unwrap(),
             None => {
                 info!("package {} has no download url, skipping", package_name);
@@ -115,12 +106,32 @@ impl Argus {
 
         p.set_message(format!("[{test_id}] testing package {package_name}",));
 
-        let path = Argus::get_path(config.clone(), pkg).await;
+        let path = Argus::get_path(config.clone(), package).await;
         p.set_message(format!(
             "testing package {package_name} -- path to download to is: {:?}",
             path
         ));
 
+        #[cfg(not(feature = "wasmer_lib"))]
+        let runner = Box::new(tester::cli_tester::CLIRunner::new(
+            test_id, config, &p, package,
+        )) as Box<dyn Tester>;
+
+        #[cfg(feature = "wasmer_lib")]
+        let runner = if config.use_lib {
+            Box::new(tester::lib_tester::LibRunner::new(
+                test_id, config, &p, package,
+            )) as Box<dyn Tester>
+        } else {
+            Box::new(tester::cli_tester::CLIRunner::new(
+                test_id, config, &p, package,
+            )) as Box<dyn Tester>
+        };
+
+        if !runner.is_to_test().await {
+            return Ok(());
+        }
+
         Argus::download_package(test_id, &path, &webc_url, &p).await?;
 
         info!("package downloaded!");
@@ -137,23 +148,11 @@ impl Argus {
         p.enable_steady_tick(Duration::from_millis(100));
 
         p.set_message("package downloaded");
-        let webc_path = path.join("package.webc");
-
-        #[cfg(feature = "wasmer_lib")]
-        let report = if config.use_lib {
-            tester::lib_tester::LibRunner::run_test(test_id, config, &p, webc_path, &package_name)
-                .await?
-        } else {
-            tester::cli_tester::CLIRunner::run_test(test_id, config, &p, webc_path, &package_name)
-                .await?
-        };
 
-        #[cfg(not(feature = "wasmer_lib"))]
-        let report =
-            tester::cli_tester::CLIRunner::run_test(test_id, config, &p, webc_path, &package_name)
-                .await?;
+        let report = runner.run_test().await?;
+        info!("\n\n\n\ntest finished\n\n\n");
 
-        Argus::write_report(&path, report, package_name.clone()).await?;
+        Argus::write_report(&path, report).await?;
 
         p.finish_with_message(format!("test for package {package_name} done!"));
         p.finish_and_clear();
@@ -162,33 +161,6 @@ impl Argus {
     }
 
     /// Checks whether or not the package should be tested
-    ///
-    /// This is done by checking if it was already tested in a compatible (i.e. same backend)
-    /// previous run by searching for the a directory with the package name in the directory
-    /// [`PackageVersionWithPackage::package`] with the same `pirita_sha256_hash` as in
-    /// [`PackageVersionWithPackage::distribution`] that contains a file that matches the current
-    /// configuration.
-    ///
-    /// For example, given a package such as
-    /// ```text
-    /// {
-    ///     "package": {
-    ///         "package_name": "any/mytest",
-    ///         ...
-    ///     },
-    ///     "distribution": {
-    ///         "pirita_sha256_hash":
-    ///             "47945b31a4169e6c82162d29e3f54cbf7cb979c8e84718a86dec1cc0f6c19890"
-    ///     }
-    ///     ...
-    /// }
-    /// ```
-    ///
-    /// this function will check if there is a file with path
-    /// `any_mytest/47945b31a4169e6c82162d29e3f54cbf7cb979c8e84718a86dec1cc0f6c19890.json`
-    /// in `outdir` as prescribed by [`Self::config`]. If the file contains a compatible test run,
-    /// it returns `false`.
-    /// If the output directory does not exists, this function returns `true`.
     async fn to_test(&self, pkg: &PackageVersionWithPackage) -> bool {
         let name = Argus::get_package_id(pkg);
 
@@ -205,57 +177,26 @@ impl Argus {
             return false;
         }
 
-        let path = Argus::get_path(Arc::new(self.config.clone()), pkg)
-            .await
-            .join("results.json");
-        if !path.exists() {
-            return true;
-        }
-
-        let file = match File::open(path) {
-            Ok(file) => file,
-            Err(e) => {
-                info!(
-                    "re-running test for pkg {:?} as previous-run file failed to open: {e}",
-                    pkg
-                );
-                return true;
-            }
-        };
-
-        let reader = BufReader::new(file);
-        let prev_run: TestResults = match serde_json::from_reader(reader) {
-            Ok(p) => p,
-            Err(e) => {
-                info!(
-                    "re-running test for pkg {:?} as previous-run file failed to be deserialized: {e}",
-                    pkg
-                );
-                return true;
-            }
-        };
-
-        !prev_run.has(&self.config)
+        return true;
     }
 
-    async fn write_report(path: &Path, report: TestReport, pkg_id: String) -> anyhow::Result<()> {
-        let test_results_path = path.join("results.json");
-
-        let mut test_results = if test_results_path.exists() {
-            let s = tokio::fs::read_to_string(&test_results_path).await?;
-            serde_json::from_str(&s)?
-        } else {
-            TestResults::from_package_id(pkg_id)
-        };
-
-        test_results.add(report);
+    #[tracing::instrument]
+    async fn write_report(path: &Path, result: TestReport) -> anyhow::Result<()> {
+        let test_results_path = path.join(format!(
+            "result-{}-{}--{}-{}.json",
+            result.runner_id,
+            result.runner_version,
+            std::env::consts::ARCH,
+            std::env::consts::OS,
+        ));
 
         let mut file = OpenOptions::new()
             .write(true)
             .create(true)
             .truncate(true)
             .open(&test_results_path)?;
-        file.write_all(serde_json::to_string(&test_results).unwrap().as_bytes())?;
+
+        file.write_all(serde_json::to_string(&result).unwrap().as_bytes())?;
         Ok(())
     }
 }
diff --git a/tests/wasmer-argus/src/argus/packages.rs b/tests/wasmer-argus/src/argus/packages.rs
index f7eff90a568..5768e7ac258 100644
--- a/tests/wasmer-argus/src/argus/packages.rs
+++ b/tests/wasmer-argus/src/argus/packages.rs
@@ -19,6 +19,7 @@ impl Argus {
         &self,
         s: UnboundedSender<PackageVersionWithPackage>,
         p: ProgressBar,
+        config: Arc<ArgusConfig>,
     ) -> anyhow::Result<()> {
         info!("starting to fetch packages..");
         let vars = AllPackageVersionsVars {
diff --git a/tests/wasmer-argus/src/argus/result.rs b/tests/wasmer-argus/src/argus/result.rs
deleted file mode 100644
index b061cc2e1dd..00000000000
--- a/tests/wasmer-argus/src/argus/result.rs
+++ /dev/null
@@ -1,70 +0,0 @@
-use crate::ArgusConfig;
-use serde::{Deserialize, Serialize};
-use std::{collections::HashMap, time::Duration};
-
-type WasmerVersion = String;
-type EngineId = String;
-
-/// The result of a test run
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
-pub struct TestResults {
-    package_id: String,
-    results: HashMap<WasmerVersion, HashMap<EngineId, TestReport>>,
-}
-
-impl TestResults {
-    pub(crate) fn has(&self, config: &ArgusConfig) -> bool {
-        let wasmer_version = config.wasmer_version();
-        let engine_id = config.compiler_backend.to_string();
-
-        if let Some(v) = self.results.get(&wasmer_version) {
-            if let Some(v) = v.get(&engine_id) {
-                return v.config.is_compatible(config);
-            }
-        }
-
-        false
-    }
-
-    pub(crate) fn add(&mut self, report: TestReport) {
-        let wasmer_version = report.config.wasmer_version();
-        let engine_id = report.config.compiler_backend.to_string();
-
-        match self.results.get_mut(&wasmer_version) {
-            Some(v) => _ = v.insert(engine_id, report),
-            None => {
-                _ = self.results.insert(
-                    wasmer_version,
-                    HashMap::from_iter(vec![(engine_id, report)]),
-                )
-            }
-        }
-    }
-
-    pub(crate) fn from_package_id(package_id: String) -> Self {
-        Self {
-            package_id,
-            ..Default::default()
-        }
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct TestReport {
-    /// The outcome of the test
-    result: Result<(), String>,
-    /// How long the test took
-    time: Duration,
-    /// The configuration of the test
-    config: ArgusConfig,
-}
-
-impl TestReport {
-    pub fn new(config: &ArgusConfig, result: Result<(), String>, time: Duration) -> Self {
-        Self {
-            result,
-            time,
-            config: config.clone(),
-        }
-    }
-}
diff --git a/tests/wasmer-argus/src/argus/tester/cli_tester.rs b/tests/wasmer-argus/src/argus/tester/cli_tester.rs
index f140b6e6817..8cbfa766ed2 100644
--- a/tests/wasmer-argus/src/argus/tester/cli_tester.rs
+++ b/tests/wasmer-argus/src/argus/tester/cli_tester.rs
@@ -1,56 +1,40 @@
-use super::Tester;
-use crate::{argus::result::TestReport, Backend};
+use crate::argus::{Argus, ArgusConfig, Backend};
 use indicatif::ProgressBar;
-use std::{path::Path, path::PathBuf, process::Command};
+use std::{fs::File, io::BufReader, path::Path, process::Command, sync::Arc};
 use tokio::time::{self, Instant};
 use tracing::*;
+use wasmer_api::types::PackageVersionWithPackage;
 use webc::{
     v1::{ParseOptions, WebCOwned},
     v2::read::OwnedReader,
     Container, Version,
 };
 
+use super::{TestReport, Tester};
+
 #[allow(unused)]
-pub struct CLIRunner {
+pub struct CLIRunner<'a> {
     test_id: u64,
-    config: std::sync::Arc<crate::ArgusConfig>,
-    webc_path: PathBuf,
-    package_name: String,
-    start_time: Instant,
+    config: Arc<ArgusConfig>,
+    p: &'a ProgressBar,
+    package: &'a PackageVersionWithPackage,
 }
 
-impl CLIRunner {
+impl<'a> CLIRunner<'a> {
     pub fn new(
         test_id: u64,
-        config: std::sync::Arc<crate::ArgusConfig>,
-        webc_path: PathBuf,
-        package_name: &str,
+        config: Arc<ArgusConfig>,
+        p: &'a ProgressBar,
+        package: &'a PackageVersionWithPackage,
     ) -> Self {
         Self {
             test_id,
             config,
-            webc_path,
-            package_name: package_name.to_string(),
-            start_time: time::Instant::now(),
+            p,
+            package,
         }
     }
 
-    pub fn ok(&self) -> anyhow::Result<TestReport> {
-        Ok(TestReport::new(
-            &self.config,
-            Ok(()),
-            time::Instant::now() - self.start_time,
-        ))
-    }
-
-    pub fn err(&self, message: String) -> anyhow::Result<TestReport> {
-        Ok(TestReport::new(
-            &self.config,
-            Err(message),
-            time::Instant::now() - self.start_time,
-        ))
-    }
-
     async fn test_atom(
         &self,
         cli_path: &String,
@@ -91,7 +75,7 @@ impl CLIRunner {
 
                 let out = cmd.output();
 
-                info!("run cmd that gave result: {:?}", out);
+                info!("run cmd that gave result: {:#?}", out);
 
                 out
             }) {
@@ -103,31 +87,83 @@ impl CLIRunner {
             },
         )
     }
+
+    fn ok(&self, version: String, start_time: Instant) -> anyhow::Result<TestReport> {
+        Ok(TestReport::new(
+            self.package,
+            String::from("wasmer_cli"),
+            version,
+            self.config.compiler_backend.to_string(),
+            start_time - Instant::now(),
+            Ok(String::from("test passed")),
+        ))
+    }
+
+    fn err(
+        &self,
+        version: String,
+        start_time: Instant,
+        message: String,
+    ) -> anyhow::Result<TestReport> {
+        Ok(TestReport::new(
+            self.package,
+            String::from("wasmer_cli"),
+            version,
+            self.config.compiler_backend.to_string(),
+            start_time - Instant::now(),
+            Err(message),
+        ))
+    }
+
+    fn get_id(&self) -> String {
+        String::from("wasmer_cli")
+    }
+
+    async fn get_version(&self) -> anyhow::Result<String> {
+        let cli_path = match &self.config.cli_path {
+            Some(ref p) => p.clone(),
+            None => String::from("wasmer"),
+        };
+
+        let mut cmd = Command::new(&cli_path);
+        let cmd = cmd.arg("-V");
+
+        info!("running cmd: {:?}", cmd);
+
+        let out = cmd.output();
+
+        info!("run cmd that gave result: {:?}", out);
+
+        match out {
+            Ok(v) => Ok(String::from_utf8(v.stdout)
+                .unwrap()
+                .replace(' ', "")
+                .replace("wasmer", "")
+                .trim()
+                .to_string()),
+            Err(e) => anyhow::bail!("failed to launch cli program {cli_path}: {e}"),
+        }
+    }
 }
 
 #[async_trait::async_trait]
-impl Tester for CLIRunner {
-    async fn run_test(
-        test_id: u64,
-        config: std::sync::Arc<crate::ArgusConfig>,
-        p: &ProgressBar,
-        webc_path: PathBuf,
-        package_name: &str,
-    ) -> anyhow::Result<crate::argus::result::TestReport> {
-        let runner = CLIRunner::new(test_id, config, webc_path, package_name);
-
-        let cli_path = match &runner.config.cli_path {
+impl<'a> Tester for CLIRunner<'a> {
+    async fn run_test(&self) -> anyhow::Result<TestReport> {
+        let start_time = time::Instant::now();
+        let version = self.get_version().await?;
+        let cli_path = match &self.config.cli_path {
             Some(ref p) => p.clone(),
             None => String::from("wasmer"),
         };
 
         info!("starting test using CLI at {cli_path}");
-        let mut dir_path = runner.webc_path.clone();
-        dir_path.pop();
+        let dir_path = Argus::get_path(self.config.clone(), self.package).await;
+        let webc_path = dir_path.join("package.webc");
 
-        p.set_message(format!("unpacking webc at {:?}", runner.webc_path));
+        self.p
+            .set_message(format!("unpacking webc at {:?}", webc_path));
 
-        let bytes = std::fs::read(&runner.webc_path)?;
+        let bytes = std::fs::read(&webc_path)?;
 
         let webc = match webc::detect(bytes.as_slice()) {
             Ok(Version::V1) => {
@@ -136,20 +172,67 @@ impl Tester for CLIRunner {
                 Container::from(webc)
             }
             Ok(Version::V2) => Container::from(OwnedReader::parse(bytes)?),
-            Ok(other) => return runner.err(format!("Unsupported version, {other}")),
-            Err(e) => return runner.err(format!("An error occurred: {e}")),
+            Ok(other) => {
+                return self.err(version, start_time, format!("Unsupported version, {other}"))
+            }
+            Err(e) => return self.err(version, start_time, format!("An error occurred: {e}")),
         };
 
         for (i, atom) in webc.atoms().iter().enumerate() {
-            p.set_message(format!("testing atom #{i}"));
-            if let Err(e) = runner
+            self.p.set_message(format!("testing atom #{i}"));
+            if let Err(e) = self
                 .test_atom(&cli_path, atom.1.as_slice(), &dir_path, i)
                 .await?
             {
-                return runner.err(e);
+                return self.err(version, start_time, e);
             }
         }
 
-        runner.ok()
+        self.ok(version, start_time)
+    }
+
+    async fn is_to_test(&self) -> bool {
+        let pkg = self.package;
+        let version = match self.get_version().await {
+            Ok(version) => version,
+            Err(e) => {
+                error!("skipping test because of error while spawning wasmer CLI command: {e}");
+                return false;
+            }
+        };
+
+        let out_dir = Argus::get_path(self.config.clone(), self.package).await;
+        let test_results_path = out_dir.join(format!(
+            "result-{}-{}--{}-{}.json",
+            self.get_id(),
+            version,
+            std::env::consts::ARCH,
+            std::env::consts::OS,
+        ));
+
+        let file = match File::open(test_results_path) {
+            Ok(file) => file,
+            Err(e) => {
+                info!(
+                    "re-running test for pkg {:?} as previous-run file failed to open: {e}",
+                    pkg
+                );
+                return true;
+            }
+        };
+
+        let reader = BufReader::new(file);
+        let report: TestReport = match serde_json::from_reader(reader) {
+            Ok(p) => p,
+            Err(e) => {
+                info!(
+                    "re-running test for pkg {:?} as previous-run file failed to be deserialized: {e}",
+                    pkg
+                );
+                return true;
+            }
+        };
+
+        report.to_test(self.config.clone())
     }
 }
diff --git a/tests/wasmer-argus/src/argus/tester/lib_tester.rs b/tests/wasmer-argus/src/argus/tester/lib_tester.rs
index 63963e3b15a..6adb5a2d5a1 100644
--- a/tests/wasmer-argus/src/argus/tester/lib_tester.rs
+++ b/tests/wasmer-argus/src/argus/tester/lib_tester.rs
@@ -1,33 +1,81 @@
-use super::Tester;
-use crate::{argus::result::TestReport, Backend};
+use super::{TestReport, Tester};
+use crate::argus::{Argus, ArgusConfig, Backend};
 use indicatif::ProgressBar;
-use std::path::PathBuf;
+use std::{fs::File, io::BufReader, sync::Arc};
 use tokio::time;
 use tracing::*;
 use wasmer::{sys::Features, Engine, NativeEngineExt, Target};
+use wasmer_api::types::PackageVersionWithPackage;
 use webc::{
     v1::{ParseOptions, WebCOwned},
     v2::read::OwnedReader,
     Container, Version,
 };
 
-pub struct LibRunner;
+pub struct LibRunner<'a> {
+    pub test_id: u64,
+    pub config: Arc<ArgusConfig>,
+    pub p: &'a ProgressBar,
+    pub package: &'a PackageVersionWithPackage,
+}
+
+impl<'a> LibRunner<'a> {
+    pub fn new(
+        test_id: u64,
+        config: Arc<ArgusConfig>,
+        p: &'a ProgressBar,
+        package: &'a PackageVersionWithPackage,
+    ) -> Self {
+        Self {
+            test_id,
+            config,
+            p,
+            package,
+        }
+    }
+
+    pub fn backend_to_engine(backend: &Backend) -> Engine {
+        match backend {
+            Backend::Llvm => Engine::new(
+                Box::new(wasmer::LLVM::new()),
+                Target::default(),
+                Features::default(),
+            ),
+            Backend::Singlepass => Engine::new(
+                Box::new(wasmer::Singlepass::new()),
+                Target::default(),
+                Features::default(),
+            ),
+            Backend::Cranelift => Engine::new(
+                Box::new(wasmer::Cranelift::new()),
+                Target::default(),
+                Features::default(),
+            ),
+        }
+    }
+
+    fn get_id(&self) -> String {
+        String::from("wasmer_lib")
+    }
+
+    fn get_version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+}
 
 #[async_trait::async_trait]
-impl Tester for LibRunner {
-    async fn run_test(
-        _test_id: u64,
-        config: std::sync::Arc<crate::ArgusConfig>,
-        p: &ProgressBar,
-        webc_path: PathBuf,
-        package_name: &str,
-    ) -> anyhow::Result<crate::argus::result::TestReport> {
+impl<'a> Tester for LibRunner<'a> {
+    async fn run_test(&self) -> anyhow::Result<TestReport> {
+        let package_id = crate::Argus::get_package_id(self.package);
+
         let start = time::Instant::now();
+        let dir_path = Argus::get_path(self.config.clone(), self.package).await;
+        let webc_path = dir_path.join("package.webc");
 
         let test_exec_result = std::panic::catch_unwind(|| {
-            p.set_message("reading webc bytes from filesystem");
+            self.p.set_message("reading webc bytes from filesystem");
             let bytes = std::fs::read(&webc_path)?;
-            let store = wasmer::Store::new(Self::backend_to_engine(&config.compiler_backend));
+            let store = wasmer::Store::new(Self::backend_to_engine(&self.config.compiler_backend));
 
             let webc = match webc::detect(bytes.as_slice()) {
                 Ok(Version::V1) => {
@@ -40,7 +88,7 @@ impl Tester for LibRunner {
                 Err(e) => anyhow::bail!("An error occurred: {e}"),
             };
 
-            p.set_message("created webc");
+            self.p.set_message("created webc");
 
             for atom in webc.atoms().iter() {
                 info!(
@@ -48,8 +96,8 @@ impl Tester for LibRunner {
                     atom.0,
                     atom.1.len()
                 );
-                p.set_message(format!(
-                    "-- {package_name} -- creating module for atom {} (has length {} bytes)",
+                self.p.set_message(format!(
+                    "[{package_id}] creating module for atom {} (has length {} bytes)",
                     atom.0,
                     atom.1.len()
                 ));
@@ -59,38 +107,59 @@ impl Tester for LibRunner {
             Ok(())
         });
 
-        let res = match test_exec_result {
+        let outcome = match test_exec_result {
             Ok(r) => match r {
-                Ok(_) => Ok(()),
+                Ok(_) => Ok(String::from("test passed")),
                 Err(e) => Err(format!("{e}")),
             },
             Err(e) => Err(format!("{:?}", e)),
         };
 
-        let time = start - time::Instant::now();
-
-        Ok(TestReport::new(config.as_ref(), res, time))
+        Ok(TestReport::new(
+            self.package,
+            self.get_id(),
+            self.get_version(),
+            self.config.compiler_backend.to_string(),
+            start - time::Instant::now(),
+            outcome,
+        ))
     }
-}
 
-impl LibRunner {
-    pub fn backend_to_engine(backend: &Backend) -> Engine {
-        match backend {
-            Backend::Llvm => Engine::new(
-                Box::new(wasmer::LLVM::new()),
-                Target::default(),
-                Features::default(),
-            ),
-            Backend::Singlepass => Engine::new(
-                Box::new(wasmer::Singlepass::new()),
-                Target::default(),
-                Features::default(),
-            ),
-            Backend::Cranelift => Engine::new(
-                Box::new(wasmer::Cranelift::new()),
-                Target::default(),
-                Features::default(),
-            ),
-        }
+    async fn is_to_test(&self) -> bool {
+        let pkg = self.package;
+
+        let out_dir = Argus::get_path(self.config.clone(), self.package).await;
+        let test_results_path = out_dir.join(format!(
+            "result-{}-{}--{}-{}.json",
+            self.get_id(),
+            self.get_version(),
+            std::env::consts::ARCH,
+            std::env::consts::OS,
+        ));
+
+        let file = match File::open(test_results_path) {
+            Ok(file) => file,
+            Err(e) => {
+                info!(
+                    "re-running test for pkg {:?} as previous-run file failed to open: {e}",
+                    pkg
+                );
+                return true;
+            }
+        };
+
+        let reader = BufReader::new(file);
+        let report: TestReport = match serde_json::from_reader(reader) {
+            Ok(p) => p,
+            Err(e) => {
+                info!(
+                    "re-running test for pkg {:?} as previous-run file failed to be deserialized: {e}",
+                    pkg
+                );
+                return true;
+            }
+        };
+
+        report.to_test(self.config.clone())
     }
 }
diff --git a/tests/wasmer-argus/src/argus/tester/mod.rs b/tests/wasmer-argus/src/argus/tester/mod.rs
index 746b906adc2..ea75446d068 100644
--- a/tests/wasmer-argus/src/argus/tester/mod.rs
+++ b/tests/wasmer-argus/src/argus/tester/mod.rs
@@ -1,7 +1,8 @@
-use super::result::TestReport;
-use crate::ArgusConfig;
-use indicatif::ProgressBar;
-use std::{path::PathBuf, sync::Arc};
+use serde::{Deserialize, Serialize};
+use std::{sync::Arc, time::Duration};
+use wasmer_api::types::PackageVersionWithPackage;
+
+use super::ArgusConfig;
 
 pub(crate) mod cli_tester;
 
@@ -9,12 +10,58 @@ pub(crate) mod cli_tester;
 pub(crate) mod lib_tester;
 
 #[async_trait::async_trait]
-pub(crate) trait Tester {
-    async fn run_test(
-        test_id: u64,
-        config: Arc<ArgusConfig>,
-        p: &ProgressBar,
-        webc_path: PathBuf,
-        package_name: &str,
-    ) -> anyhow::Result<TestReport>;
+pub(crate) trait Tester: Send + Sync {
+    async fn is_to_test(&self) -> bool;
+    async fn run_test(&self) -> anyhow::Result<TestReport>;
+}
+
+/// The result of a test run.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TestReport {
+    pub package_namespace: String,
+    pub package_name: String,
+    pub package_version: String,
+
+    /// The unique identifier of the test runner.
+    ///
+    /// In practice, it will be one of `wasmer_cli`
+    /// or `wasmer_lib`.
+    pub runner_id: String,
+    pub runner_version: String,
+
+    /// The unique identifier of the compiler backend used to perform the test.
+    pub compiler_backend: String,
+
+    pub time: Duration,
+    pub outcome: Result<String, String>,
+}
+
+impl TestReport {
+    pub fn new(
+        package: &PackageVersionWithPackage,
+        runner_id: String,
+        runner_version: String,
+        compiler_backend: String,
+        time: Duration,
+        outcome: Result<String, String>,
+    ) -> Self {
+        Self {
+            package_namespace: match &package.package.namespace {
+                Some(ns) => ns.clone(),
+                None => String::from("unknown_namespace"),
+            },
+            package_name: package.package.package_name.clone(),
+            package_version: package.version.clone(),
+            runner_id,
+            runner_version,
+            compiler_backend,
+            time,
+            outcome,
+        }
+    }
+
+    pub fn to_test(&self, _config: Arc<ArgusConfig>) -> bool {
+        // In time we will have more checks to add here.
+        true
+    }
 }

From 1221ddfe59c44267ea18b8b51b9beb4d65ad983d Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Wed, 20 Mar 2024 20:48:31 +0100
Subject: [PATCH 30/33] chore(argus): clippy + fmt

---
 tests/wasmer-argus/src/argus/config.rs | 1 -
 tests/wasmer-argus/src/argus/mod.rs    | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/wasmer-argus/src/argus/config.rs b/tests/wasmer-argus/src/argus/config.rs
index 7651f56bf08..5a325f0e769 100644
--- a/tests/wasmer-argus/src/argus/config.rs
+++ b/tests/wasmer-argus/src/argus/config.rs
@@ -57,7 +57,6 @@ pub struct ArgusConfig {
     #[arg(long)]
     pub cli_path: Option<String>,
 
-
     /// Whether or not this run should use the linked [`wasmer-api`] library instead of the CLI.
     #[cfg(feature = "wasmer_lib")]
     #[arg(long, conflicts_with = "cli_path")]
diff --git a/tests/wasmer-argus/src/argus/mod.rs b/tests/wasmer-argus/src/argus/mod.rs
index 82673428a6b..073dc460e65 100644
--- a/tests/wasmer-argus/src/argus/mod.rs
+++ b/tests/wasmer-argus/src/argus/mod.rs
@@ -177,7 +177,7 @@ impl Argus {
             return false;
         }
 
-        return true;
+        true
     }
 
     #[tracing::instrument]
@@ -194,7 +194,7 @@ impl Argus {
             .write(true)
             .create(true)
             .truncate(true)
-            .open(&test_results_path)?;
+            .open(test_results_path)?;
 
         file.write_all(serde_json::to_string(&result).unwrap().as_bytes())?;
         Ok(())

From 2db889fe4e98b6358566053fe556706dbfe944a2 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Thu, 21 Mar 2024 14:47:21 +0100
Subject: [PATCH 31/33] fix(argus): use rustls in reqwest

---
 tests/wasmer-argus/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/wasmer-argus/Cargo.toml b/tests/wasmer-argus/Cargo.toml
index a0306e10cbb..aea5bf8a47b 100644
--- a/tests/wasmer-argus/Cargo.toml
+++ b/tests/wasmer-argus/Cargo.toml
@@ -25,7 +25,7 @@ tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 serde = { version = "1.0.197", features = ["derive"] }
 serde_json = "1.0.114"
 wasmer = { version = "4.2.6", path = "../../lib/api", features = ["engine", "core", "singlepass", "cranelift", "llvm"], optional = true }
-reqwest = "0.11.24"
+reqwest = { version = "0.11.24", features = ["rustls-tls"], default-features = false }
 derive_more = "0.99.17"
 webc.workspace = true
 async-trait = "0.1.77"

From e73190ca26b1d0519f93b1e8b4298f57863f50d5 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Thu, 21 Mar 2024 14:53:36 +0100
Subject: [PATCH 32/33] fix(argus): use same reqwest config as CLI

---
 tests/wasmer-argus/Cargo.toml | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/tests/wasmer-argus/Cargo.toml b/tests/wasmer-argus/Cargo.toml
index aea5bf8a47b..2946a6b04c3 100644
--- a/tests/wasmer-argus/Cargo.toml
+++ b/tests/wasmer-argus/Cargo.toml
@@ -25,8 +25,23 @@ tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 serde = { version = "1.0.197", features = ["derive"] }
 serde_json = "1.0.114"
 wasmer = { version = "4.2.6", path = "../../lib/api", features = ["engine", "core", "singlepass", "cranelift", "llvm"], optional = true }
-reqwest = { version = "0.11.24", features = ["rustls-tls"], default-features = false }
 derive_more = "0.99.17"
 webc.workspace = true
 async-trait = "0.1.77"
 wasmer-api = { path = "../../lib/backend-api" }
+
+
+[target.'cfg(not(target_arch = "riscv64"))'.dependencies]
+reqwest = { version = "^0.11", default-features = false, features = [
+  "rustls-tls",
+  "json",
+  "multipart",
+  "gzip",
+] }
+
+[target.'cfg(target_arch = "riscv64")'.dependencies]
+reqwest = { version = "^0.11", default-features = false, features = [
+  "native-tls",
+  "json",
+  "multipart",
+] }

From 6aae478125f3b57edd79615c1d63c74c86548d92 Mon Sep 17 00:00:00 2001
From: Edoardo Marangoni <ecmm@anche.no>
Date: Thu, 21 Mar 2024 15:14:16 +0100
Subject: [PATCH 33/33] log version

---
 Cargo.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index d18df5d0ed4..9068cad41db 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6218,7 +6218,7 @@ dependencies = [
  "derive_more",
  "futures 0.3.30",
  "indicatif",
- "log",
+ "log 0.4.21",
  "reqwest",
  "serde",
  "serde_json",