diff --git a/Cargo.lock b/Cargo.lock index affbb8769ff..1503414fe24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -729,6 +729,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + [[package]] name = "cooked-waker" version = "5.0.0" @@ -1267,8 +1273,10 @@ version = "0.99.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ + "convert_case", "proc-macro2", "quote", + "rustc_version 0.4.0", "syn 1.0.109", ] @@ -6199,6 +6207,30 @@ dependencies = [ "webc", ] +[[package]] +name = "wasmer-argus" +version = "4.2.7" +dependencies = [ + "anyhow", + "async-trait", + "clap", + "cynic", + "derive_more", + "futures 0.3.30", + "indicatif", + "log 0.4.21", + "reqwest", + "serde", + "serde_json", + "tokio 1.36.0", + "tracing", + "tracing-subscriber", + "url", + "wasmer", + "wasmer-api", + "webc", +] + [[package]] name = "wasmer-bin-fuzz" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index 4acae2e943b..3b56c2af55a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,6 +70,7 @@ members = [ "tests/lib/compiler-test-derive", "tests/lib/wast", "tests/wasi-wast", + "tests/wasmer-argus", ] resolver = "2" diff --git a/tests/wasmer-argus/Cargo.toml b/tests/wasmer-argus/Cargo.toml new file mode 100644 index 00000000000..2946a6b04c3 --- /dev/null +++ b/tests/wasmer-argus/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "wasmer-argus" +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[features] +wasmer_lib = ["dep:wasmer"] + +[dependencies] +indicatif = "0.17.8" +anyhow = "1.0.80" +log = "0.4.21" +cynic = "3.4.3" +url = "2.5.0" +futures = "0.3.30" +tracing = "0.1.40" +tokio = { version = "1.36.0", features = ["rt-multi-thread", "sync", "time", "fs"] } +clap = {version = "4.4.11", features = ["derive", "string"]} +tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } +serde = { version = "1.0.197", features = ["derive"] } +serde_json = "1.0.114" +wasmer = { version = "4.2.6", path = "../../lib/api", features = ["engine", "core", "singlepass", "cranelift", "llvm"], optional = true } +derive_more = "0.99.17" +webc.workspace = true +async-trait = "0.1.77" +wasmer-api = { path = "../../lib/backend-api" } + + +[target.'cfg(not(target_arch = "riscv64"))'.dependencies] +reqwest = { version = "^0.11", default-features = false, features = [ + "rustls-tls", + "json", + "multipart", + "gzip", +] } + +[target.'cfg(target_arch = "riscv64")'.dependencies] +reqwest = { version = "^0.11", default-features = false, features = [ + "native-tls", + "json", + "multipart", +] } diff --git a/tests/wasmer-argus/README.md b/tests/wasmer-argus/README.md new file mode 100644 index 00000000000..f319986a306 --- /dev/null +++ b/tests/wasmer-argus/README.md @@ -0,0 +1,55 @@ +# wasmer-argus + +Automatically test packages from the registry. + +## Building +If you want to use the local `wasmer` crate, you shall +build the project with `cargo build --package wasmer-argus --features wasmer_lib`. + +On macOS, you may encounter an error where the linker does not find `zstd`: a possible +solution to this problem is to install `zstd` using `brew` (`brew install zstd`) and +using the following command: + +`RUSTFLAGS="-L$(brew --prefix)/lib" cargo build --package wasmer-argus --features wasmer_lib` + +Another possiblity is to add the your brew prefix with `/lib` (probably = `/opt/homebrew/lib/`) +to the global Cargo config something like: +``` +[target.aarch64-apple-darwin] +rustflags = ["-L/opt/homebrew/lib"] +``` + +## Usage +This binary fetches packages from the graphql endpoint of a registry. By +default, it uses `http://registry.wasmer.io/graphql`; and the needed +authorization token is retrieved from the environment using the `WASMER_TOKEN`. +Users can specify the token via CLI with the appropriate flag. + +This testsuite is parallelised, and the degree of parallelism available can be +specified both by CLI flag or automatically using +`std::thread::available_parallelism`. + +``` +Fetch and test packages from a WebContainer registry + +Usage: wasmer-argus [OPTIONS] + +Options: + -r, --registry-url + The GraphQL endpoint of the registry to test [default: http://registry.wasmer.io/graphql] + -b, --backend + The backend to test the compilation against [default: singlepass] [possible values: llvm, singlepass, cranelift] + --run + Whether or not to run packages during tests + -o, --outdir + The output directory [default: /home/ecmm/sw/wasmer/wasmer/target/debug/out] + --auth-token + The authorization token needed to see packages [default: ] + --jobs + The number of concurrent tests (jobs) to perform [default: 12] + -h, --help Print help + -V, --version Print version +``` + + + diff --git a/tests/wasmer-argus/src/argus/config.rs b/tests/wasmer-argus/src/argus/config.rs new file mode 100644 index 00000000000..5a325f0e769 --- /dev/null +++ b/tests/wasmer-argus/src/argus/config.rs @@ -0,0 +1,64 @@ +use clap::ValueEnum; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +fn get_default_out_path() -> PathBuf { + let mut path = std::env::current_dir().unwrap(); + path.push("out"); + path +} + +fn get_default_token() -> String { + std::env::var("WASMER_TOKEN").unwrap_or_default() +} + +fn get_default_jobs() -> usize { + std::thread::available_parallelism() + .unwrap_or(std::num::NonZeroUsize::new(2).unwrap()) + .into() +} + +#[derive(Debug, Clone, Serialize, Deserialize, ValueEnum, derive_more::Display)] +pub enum Backend { + Llvm, + Singlepass, + Cranelift, +} + +/// Fetch and test packages from a WebContainer registry. +#[derive(Debug, clap::Parser, Clone, Serialize, Deserialize)] +#[command(version, about, long_about = None)] +pub struct ArgusConfig { + /// The GraphQL endpoint of the registry to test + #[arg( + short, + long, + default_value_t = String::from("http://registry.wasmer.io/graphql") + )] + pub registry_url: String, + + /// The backend to test the compilation against + #[arg(short = 'b', long = "backend", value_enum, default_value_t = Backend::Singlepass)] + pub compiler_backend: Backend, + + /// The output directory + #[arg(short = 'o', long, default_value = get_default_out_path().into_os_string())] + pub outdir: std::path::PathBuf, + + /// The authorization token needed to see packages + #[arg(long, default_value_t = get_default_token())] + pub auth_token: String, + + /// The number of concurrent tests (jobs) to perform + #[arg(long, default_value_t = get_default_jobs()) ] + pub jobs: usize, + + /// The path to the CLI command to use. [default will be searched in $PATH: "wasmer"] + #[arg(long)] + pub cli_path: Option, + + /// Whether or not this run should use the linked [`wasmer-api`] library instead of the CLI. + #[cfg(feature = "wasmer_lib")] + #[arg(long, conflicts_with = "cli_path")] + pub use_lib: bool, +} diff --git a/tests/wasmer-argus/src/argus/mod.rs b/tests/wasmer-argus/src/argus/mod.rs new file mode 100644 index 00000000000..073dc460e65 --- /dev/null +++ b/tests/wasmer-argus/src/argus/mod.rs @@ -0,0 +1,202 @@ +mod config; +mod packages; +mod tester; + +use self::tester::{TestReport, Tester}; +pub use config::*; +use indicatif::{MultiProgress, ProgressBar}; +use std::{fs::OpenOptions, io::Write as _, path::Path, sync::Arc, time::Duration}; +use tokio::{ + sync::{mpsc, Semaphore}, + task::JoinSet, +}; +use tracing::*; +use url::Url; +use wasmer_api::{types::PackageVersionWithPackage, WasmerClient}; + +#[derive(Debug, Clone)] +pub struct Argus { + pub config: ArgusConfig, + pub client: WasmerClient, +} + +impl TryFrom for Argus { + type Error = anyhow::Error; + + fn try_from(config: ArgusConfig) -> Result { + let client = WasmerClient::new(Url::parse(&config.registry_url)?, "wasmer-argus")?; + + let client = client.with_auth_token(config.auth_token.clone()); + Ok(Argus { client, config }) + } +} + +impl Argus { + /// Start the testsuite using the configuration in [`Self::config`] + pub async fn run(self) -> anyhow::Result<()> { + info!("fetching packages from {}", self.config.registry_url); + + let m = MultiProgress::new(); + let (s, mut r) = mpsc::unbounded_channel(); + + let mut pool = JoinSet::new(); + let c = Arc::new(self.config.clone()); + + { + let this = self.clone(); + let bar = m.add(ProgressBar::new(0)); + + pool.spawn(async move { this.fetch_packages(s, bar, c.clone()).await }); + } + + let mut count = 0; + + let c = Arc::new(self.config.clone()); + let sem = Arc::new(Semaphore::new(self.config.jobs)); + + while let Some(pkg) = r.recv().await { + let c = c.clone(); + let bar = m.add(ProgressBar::new(0)); + let permit = Arc::clone(&sem).acquire_owned().await; + + pool.spawn(async move { + let _permit = permit; + Argus::test(count, c, &pkg, bar).await + }); + + count += 1; + } + + while let Some(t) = pool.join_next().await { + if let Err(e) = t { + error!("task failed: {e}") + } + } + + info!("done!"); + Ok(()) + } + + /// Perform the test for a single package + async fn test( + test_id: u64, + config: Arc, + package: &PackageVersionWithPackage, + p: ProgressBar, + ) -> anyhow::Result<()> { + p.set_style( + indicatif::ProgressStyle::with_template(&format!( + "[{test_id}] {{spinner:.blue}} {{msg}}" + )) + .unwrap() + .tick_strings(&["✶", "✸", "✹", "✺", "✹", "✷"]), + ); + + p.enable_steady_tick(Duration::from_millis(100)); + + let package_name = Argus::get_package_id(package); + let webc_url: Url = match &package.distribution.pirita_download_url { + Some(url) => url.parse().unwrap(), + None => { + info!("package {} has no download url, skipping", package_name); + p.finish_and_clear(); + return Ok(()); + } + }; + + p.set_message(format!("[{test_id}] testing package {package_name}",)); + + let path = Argus::get_path(config.clone(), package).await; + p.set_message(format!( + "testing package {package_name} -- path to download to is: {:?}", + path + )); + + #[cfg(not(feature = "wasmer_lib"))] + let runner = Box::new(tester::cli_tester::CLIRunner::new( + test_id, config, &p, package, + )) as Box; + + #[cfg(feature = "wasmer_lib")] + let runner = if config.use_lib { + Box::new(tester::lib_tester::LibRunner::new( + test_id, config, &p, package, + )) as Box + } else { + Box::new(tester::cli_tester::CLIRunner::new( + test_id, config, &p, package, + )) as Box + }; + + if !runner.is_to_test().await { + return Ok(()); + } + + Argus::download_package(test_id, &path, &webc_url, &p).await?; + + info!("package downloaded!"); + + p.reset(); + p.set_style( + indicatif::ProgressStyle::with_template(&format!( + "[{test_id}/{package_name}] {{spinner:.blue}} {{msg}}" + )) + .unwrap() + .tick_strings(&["✶", "✸", "✹", "✺", "✹", "✷"]), + ); + + p.enable_steady_tick(Duration::from_millis(100)); + + p.set_message("package downloaded"); + + let report = runner.run_test().await?; + info!("\n\n\n\ntest finished\n\n\n"); + + Argus::write_report(&path, report).await?; + + p.finish_with_message(format!("test for package {package_name} done!")); + p.finish_and_clear(); + + Ok(()) + } + + /// Checks whether or not the package should be tested + async fn to_test(&self, pkg: &PackageVersionWithPackage) -> bool { + let name = Argus::get_package_id(pkg); + + info!("checking if package {name} needs to be tested or not"); + + let dir_path = std::path::PathBuf::from(&self.config.outdir); + + if !dir_path.exists() { + return true; + } + + if pkg.distribution.pirita_sha256_hash.is_none() { + info!("skipping test for {name} as it has no hash"); + return false; + } + + true + } + + #[tracing::instrument] + async fn write_report(path: &Path, result: TestReport) -> anyhow::Result<()> { + let test_results_path = path.join(format!( + "result-{}-{}--{}-{}.json", + result.runner_id, + result.runner_version, + std::env::consts::ARCH, + std::env::consts::OS, + )); + + let mut file = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(test_results_path)?; + + file.write_all(serde_json::to_string(&result).unwrap().as_bytes())?; + Ok(()) + } +} diff --git a/tests/wasmer-argus/src/argus/packages.rs b/tests/wasmer-argus/src/argus/packages.rs new file mode 100644 index 00000000000..5768e7ac258 --- /dev/null +++ b/tests/wasmer-argus/src/argus/packages.rs @@ -0,0 +1,218 @@ +use super::Argus; +use crate::ArgusConfig; +use futures::StreamExt; +use indicatif::{ProgressBar, ProgressStyle}; +use reqwest::{header, Client}; +use std::{path::PathBuf, sync::Arc, time::Duration}; +use tokio::{fs::File, io::AsyncWriteExt, sync::mpsc::UnboundedSender}; +use tracing::*; +use url::Url; +use wasmer_api::{ + query::get_package_versions_stream, + types::{AllPackageVersionsVars, PackageVersionSortBy, PackageVersionWithPackage}, +}; + +impl Argus { + /// Fetch all packages from the registry + #[tracing::instrument(skip(self, s, p))] + pub async fn fetch_packages( + &self, + s: UnboundedSender, + p: ProgressBar, + config: Arc, + ) -> anyhow::Result<()> { + info!("starting to fetch packages.."); + let vars = AllPackageVersionsVars { + sort_by: Some(PackageVersionSortBy::Oldest), + ..Default::default() + }; + + p.set_style( + ProgressStyle::with_template("{spinner:.blue} {msg}") + .unwrap() + .tick_strings(&["✶", "✸", "✹", "✺", "✹", "✷"]), + ); + p.enable_steady_tick(Duration::from_millis(1000)); + + let mut count = 0; + + let call = get_package_versions_stream(&self.client, vars.clone()); + futures::pin_mut!(call); + p.set_message("starting to fetch packages..".to_string()); + + while let Some(pkgs) = call.next().await { + let pkgs = match pkgs { + Ok(pkgs) => pkgs, + Err(e) => { + error!("failed to fetch packages: {e}"); + p.finish_and_clear(); + anyhow::bail!("failed to fetch packages: {e}") + } + }; + p.set_message(format!("fetched {} packages", count)); + count += pkgs.len(); + + for pkg in pkgs { + if self.to_test(&pkg).await { + if let Err(e) = s.send(pkg) { + error!("failed to send packages: {e}"); + p.finish_and_clear(); + anyhow::bail!("failed to send packages: {e}") + }; + } + } + } + + p.finish_with_message(format!("fetched {count} packages")); + info!("finished fetching packages: fetched {count} packages, closing channel"); + drop(s); + Ok(()) + } + + #[tracing::instrument(skip(p))] + pub(crate) async fn download_package<'a>( + test_id: u64, + path: &'a PathBuf, + url: &'a Url, + p: &'a ProgressBar, + ) -> anyhow::Result<()> { + info!("downloading package from {} to file {:?}", url, path); + static APP_USER_AGENT: &str = + concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"),); + + if !path.exists() { + tokio::fs::create_dir_all(path).await?; + } else if path.exists() && !path.is_dir() { + anyhow::bail!("path {:?} exists, but it is not a directory!", path) + } + + let client = Client::builder().user_agent(APP_USER_AGENT).build()?; + + let download_size = { + let resp = client.head(url.as_str()).send().await?; + if resp.status().is_success() { + resp.headers() + .get(header::CONTENT_LENGTH) + .and_then(|ct_len| ct_len.to_str().ok()) + .and_then(|ct_len| ct_len.parse().ok()) + .unwrap_or(0) // Fallback to 0 + } else { + anyhow::bail!( + "Couldn't fetch head from URL {}. Error: {:?}", + url, + resp.status() + ) + } + }; + + let request = client.get(url.as_str()); + + p.set_length(download_size); + + p.set_style( + ProgressStyle::default_bar() + .template(&format!( + "[{test_id}] [{{bar:40.cyan/blue}}] {{bytes}}/{{total_bytes}} - {{msg}}" + )) + .unwrap() + .progress_chars("#>-"), + ); + + p.set_message(format!("downloading from {url}")); + + let mut outfile = match File::create(&path.join("package.webc")).await { + Ok(o) => o, + Err(e) => { + error!( + "[{test_id}] failed to create file at {:?}. Error: {e}", + path.join("package.webc") + ); + + p.finish_and_clear(); + + anyhow::bail!( + "[{test_id}] failed to create file at {:?}. Error: {e}", + path.join("package.webc") + ); + } + }; + let mut download = match request.send().await { + Ok(d) => d, + Err(e) => { + error!("[{test_id}] failed to download from URL {url}. Error: {e}"); + p.finish_and_clear(); + anyhow::bail!("[{test_id}] failed to download from URL {url}. Error: {e}"); + } + }; + + loop { + match download.chunk().await { + Err(e) => { + error!( + "[{test_id}] failed to download chunk from {:?}. Error: {e}", + download + ); + p.finish_and_clear(); + anyhow::bail!( + "[{test_id}] failed to download chunk from {:?}. Error: {e}", + download + ); + } + Ok(chunk) => { + if let Some(chunk) = chunk { + p.inc(chunk.len() as u64); + if let Err(e) = outfile.write(&chunk).await { + error!( + "[{test_id}] failed to write chunk to file {:?}. Error: {e}", + outfile + ); + p.finish_and_clear(); + anyhow::bail!( + "[{test_id}] failed to write chunk to file {:?}. Error: {e}", + outfile + ); + }; + } else { + break; + } + } + } + } + + outfile.flush().await?; + drop(outfile); + + Ok(()) + } + + /// Return the complete path to the folder of the test for the package, from the outdir to the + /// hash + pub async fn get_path(config: Arc, pkg: &PackageVersionWithPackage) -> PathBuf { + let hash = match &pkg.distribution.pirita_sha256_hash { + Some(hash) => hash, + None => { + unreachable!("no package without an hash should reach this function!") + } + }; + + let _namespace = match &pkg.package.namespace { + Some(ns) => ns.replace('/', "_"), + None => "unknown_namespace".to_owned(), + }; + + config.outdir.join(hash) + } + + pub fn get_package_id(pkg: &PackageVersionWithPackage) -> String { + let namespace = match &pkg.package.namespace { + Some(namespace) => namespace.replace('/', "_"), + None => String::from("unknown_namespace"), + }; + format!( + "{}/{}_v{}", + namespace, + pkg.package.package_name.replace('/', "_"), + pkg.version + ) + } +} diff --git a/tests/wasmer-argus/src/argus/tester/cli_tester.rs b/tests/wasmer-argus/src/argus/tester/cli_tester.rs new file mode 100644 index 00000000000..8cbfa766ed2 --- /dev/null +++ b/tests/wasmer-argus/src/argus/tester/cli_tester.rs @@ -0,0 +1,238 @@ +use crate::argus::{Argus, ArgusConfig, Backend}; +use indicatif::ProgressBar; +use std::{fs::File, io::BufReader, path::Path, process::Command, sync::Arc}; +use tokio::time::{self, Instant}; +use tracing::*; +use wasmer_api::types::PackageVersionWithPackage; +use webc::{ + v1::{ParseOptions, WebCOwned}, + v2::read::OwnedReader, + Container, Version, +}; + +use super::{TestReport, Tester}; + +#[allow(unused)] +pub struct CLIRunner<'a> { + test_id: u64, + config: Arc, + p: &'a ProgressBar, + package: &'a PackageVersionWithPackage, +} + +impl<'a> CLIRunner<'a> { + pub fn new( + test_id: u64, + config: Arc, + p: &'a ProgressBar, + package: &'a PackageVersionWithPackage, + ) -> Self { + Self { + test_id, + config, + p, + package, + } + } + + async fn test_atom( + &self, + cli_path: &String, + atom: &[u8], + dir_path: &Path, + atom_id: usize, + ) -> anyhow::Result> { + if let Err(e) = Command::new(cli_path).arg("-V").output() { + if let std::io::ErrorKind::NotFound = e.kind() { + anyhow::bail!("the command '{cli_path}' was not found"); + } + } + + let atom_path = dir_path.join(format!("atom_{atom_id}.wasm")); + let output_path = dir_path.join(format!("atom_{atom_id}.wasmu")); + + tokio::fs::write(&atom_path, atom).await?; + + let backend = match self.config.compiler_backend { + Backend::Llvm => "--llvm", + Backend::Singlepass => "--singlepass", + Backend::Cranelift => "--cranelift", + }; + + Ok( + match std::panic::catch_unwind(move || { + let mut cmd = Command::new(cli_path); + + let cmd = cmd.args([ + "compile", + atom_path.to_str().unwrap(), + backend, + "-o", + output_path.to_str().unwrap(), + ]); + + info!("running cmd: {:?}", cmd); + + let out = cmd.output(); + + info!("run cmd that gave result: {:#?}", out); + + out + }) { + Ok(r) => match r { + Ok(_) => Ok(()), + Err(e) => Err(e.to_string()), + }, + Err(_) => Err(String::from("thread panicked")), + }, + ) + } + + fn ok(&self, version: String, start_time: Instant) -> anyhow::Result { + Ok(TestReport::new( + self.package, + String::from("wasmer_cli"), + version, + self.config.compiler_backend.to_string(), + start_time - Instant::now(), + Ok(String::from("test passed")), + )) + } + + fn err( + &self, + version: String, + start_time: Instant, + message: String, + ) -> anyhow::Result { + Ok(TestReport::new( + self.package, + String::from("wasmer_cli"), + version, + self.config.compiler_backend.to_string(), + start_time - Instant::now(), + Err(message), + )) + } + + fn get_id(&self) -> String { + String::from("wasmer_cli") + } + + async fn get_version(&self) -> anyhow::Result { + let cli_path = match &self.config.cli_path { + Some(ref p) => p.clone(), + None => String::from("wasmer"), + }; + + let mut cmd = Command::new(&cli_path); + let cmd = cmd.arg("-V"); + + info!("running cmd: {:?}", cmd); + + let out = cmd.output(); + + info!("run cmd that gave result: {:?}", out); + + match out { + Ok(v) => Ok(String::from_utf8(v.stdout) + .unwrap() + .replace(' ', "") + .replace("wasmer", "") + .trim() + .to_string()), + Err(e) => anyhow::bail!("failed to launch cli program {cli_path}: {e}"), + } + } +} + +#[async_trait::async_trait] +impl<'a> Tester for CLIRunner<'a> { + async fn run_test(&self) -> anyhow::Result { + let start_time = time::Instant::now(); + let version = self.get_version().await?; + let cli_path = match &self.config.cli_path { + Some(ref p) => p.clone(), + None => String::from("wasmer"), + }; + + info!("starting test using CLI at {cli_path}"); + let dir_path = Argus::get_path(self.config.clone(), self.package).await; + let webc_path = dir_path.join("package.webc"); + + self.p + .set_message(format!("unpacking webc at {:?}", webc_path)); + + let bytes = std::fs::read(&webc_path)?; + + let webc = match webc::detect(bytes.as_slice()) { + Ok(Version::V1) => { + let options = ParseOptions::default(); + let webc = WebCOwned::parse(bytes, &options)?; + Container::from(webc) + } + Ok(Version::V2) => Container::from(OwnedReader::parse(bytes)?), + Ok(other) => { + return self.err(version, start_time, format!("Unsupported version, {other}")) + } + Err(e) => return self.err(version, start_time, format!("An error occurred: {e}")), + }; + + for (i, atom) in webc.atoms().iter().enumerate() { + self.p.set_message(format!("testing atom #{i}")); + if let Err(e) = self + .test_atom(&cli_path, atom.1.as_slice(), &dir_path, i) + .await? + { + return self.err(version, start_time, e); + } + } + + self.ok(version, start_time) + } + + async fn is_to_test(&self) -> bool { + let pkg = self.package; + let version = match self.get_version().await { + Ok(version) => version, + Err(e) => { + error!("skipping test because of error while spawning wasmer CLI command: {e}"); + return false; + } + }; + + let out_dir = Argus::get_path(self.config.clone(), self.package).await; + let test_results_path = out_dir.join(format!( + "result-{}-{}--{}-{}.json", + self.get_id(), + version, + std::env::consts::ARCH, + std::env::consts::OS, + )); + + let file = match File::open(test_results_path) { + Ok(file) => file, + Err(e) => { + info!( + "re-running test for pkg {:?} as previous-run file failed to open: {e}", + pkg + ); + return true; + } + }; + + let reader = BufReader::new(file); + let report: TestReport = match serde_json::from_reader(reader) { + Ok(p) => p, + Err(e) => { + info!( + "re-running test for pkg {:?} as previous-run file failed to be deserialized: {e}", + pkg + ); + return true; + } + }; + + report.to_test(self.config.clone()) + } +} diff --git a/tests/wasmer-argus/src/argus/tester/lib_tester.rs b/tests/wasmer-argus/src/argus/tester/lib_tester.rs new file mode 100644 index 00000000000..6adb5a2d5a1 --- /dev/null +++ b/tests/wasmer-argus/src/argus/tester/lib_tester.rs @@ -0,0 +1,165 @@ +use super::{TestReport, Tester}; +use crate::argus::{Argus, ArgusConfig, Backend}; +use indicatif::ProgressBar; +use std::{fs::File, io::BufReader, sync::Arc}; +use tokio::time; +use tracing::*; +use wasmer::{sys::Features, Engine, NativeEngineExt, Target}; +use wasmer_api::types::PackageVersionWithPackage; +use webc::{ + v1::{ParseOptions, WebCOwned}, + v2::read::OwnedReader, + Container, Version, +}; + +pub struct LibRunner<'a> { + pub test_id: u64, + pub config: Arc, + pub p: &'a ProgressBar, + pub package: &'a PackageVersionWithPackage, +} + +impl<'a> LibRunner<'a> { + pub fn new( + test_id: u64, + config: Arc, + p: &'a ProgressBar, + package: &'a PackageVersionWithPackage, + ) -> Self { + Self { + test_id, + config, + p, + package, + } + } + + pub fn backend_to_engine(backend: &Backend) -> Engine { + match backend { + Backend::Llvm => Engine::new( + Box::new(wasmer::LLVM::new()), + Target::default(), + Features::default(), + ), + Backend::Singlepass => Engine::new( + Box::new(wasmer::Singlepass::new()), + Target::default(), + Features::default(), + ), + Backend::Cranelift => Engine::new( + Box::new(wasmer::Cranelift::new()), + Target::default(), + Features::default(), + ), + } + } + + fn get_id(&self) -> String { + String::from("wasmer_lib") + } + + fn get_version(&self) -> String { + env!("CARGO_PKG_VERSION").to_string() + } +} + +#[async_trait::async_trait] +impl<'a> Tester for LibRunner<'a> { + async fn run_test(&self) -> anyhow::Result { + let package_id = crate::Argus::get_package_id(self.package); + + let start = time::Instant::now(); + let dir_path = Argus::get_path(self.config.clone(), self.package).await; + let webc_path = dir_path.join("package.webc"); + + let test_exec_result = std::panic::catch_unwind(|| { + self.p.set_message("reading webc bytes from filesystem"); + let bytes = std::fs::read(&webc_path)?; + let store = wasmer::Store::new(Self::backend_to_engine(&self.config.compiler_backend)); + + let webc = match webc::detect(bytes.as_slice()) { + Ok(Version::V1) => { + let options = ParseOptions::default(); + let webc = WebCOwned::parse(bytes, &options)?; + Container::from(webc) + } + Ok(Version::V2) => Container::from(OwnedReader::parse(bytes)?), + Ok(other) => anyhow::bail!("Unsupported version, {other}"), + Err(e) => anyhow::bail!("An error occurred: {e}"), + }; + + self.p.set_message("created webc"); + + for atom in webc.atoms().iter() { + info!( + "creating module for atom {} with length {}", + atom.0, + atom.1.len() + ); + self.p.set_message(format!( + "[{package_id}] creating module for atom {} (has length {} bytes)", + atom.0, + atom.1.len() + )); + wasmer::Module::new(&store, atom.1.as_slice())?; + } + + Ok(()) + }); + + let outcome = match test_exec_result { + Ok(r) => match r { + Ok(_) => Ok(String::from("test passed")), + Err(e) => Err(format!("{e}")), + }, + Err(e) => Err(format!("{:?}", e)), + }; + + Ok(TestReport::new( + self.package, + self.get_id(), + self.get_version(), + self.config.compiler_backend.to_string(), + start - time::Instant::now(), + outcome, + )) + } + + async fn is_to_test(&self) -> bool { + let pkg = self.package; + + let out_dir = Argus::get_path(self.config.clone(), self.package).await; + let test_results_path = out_dir.join(format!( + "result-{}-{}--{}-{}.json", + self.get_id(), + self.get_version(), + std::env::consts::ARCH, + std::env::consts::OS, + )); + + let file = match File::open(test_results_path) { + Ok(file) => file, + Err(e) => { + info!( + "re-running test for pkg {:?} as previous-run file failed to open: {e}", + pkg + ); + return true; + } + }; + + let reader = BufReader::new(file); + let report: TestReport = match serde_json::from_reader(reader) { + Ok(p) => p, + Err(e) => { + info!( + "re-running test for pkg {:?} as previous-run file failed to be deserialized: {e}", + pkg + ); + return true; + } + }; + + report.to_test(self.config.clone()) + } +} diff --git a/tests/wasmer-argus/src/argus/tester/mod.rs b/tests/wasmer-argus/src/argus/tester/mod.rs new file mode 100644 index 00000000000..ea75446d068 --- /dev/null +++ b/tests/wasmer-argus/src/argus/tester/mod.rs @@ -0,0 +1,67 @@ +use serde::{Deserialize, Serialize}; +use std::{sync::Arc, time::Duration}; +use wasmer_api::types::PackageVersionWithPackage; + +use super::ArgusConfig; + +pub(crate) mod cli_tester; + +#[cfg(feature = "wasmer_lib")] +pub(crate) mod lib_tester; + +#[async_trait::async_trait] +pub(crate) trait Tester: Send + Sync { + async fn is_to_test(&self) -> bool; + async fn run_test(&self) -> anyhow::Result; +} + +/// The result of a test run. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestReport { + pub package_namespace: String, + pub package_name: String, + pub package_version: String, + + /// The unique identifier of the test runner. + /// + /// In practice, it will be one of `wasmer_cli` + /// or `wasmer_lib`. + pub runner_id: String, + pub runner_version: String, + + /// The unique identifier of the compiler backend used to perform the test. + pub compiler_backend: String, + + pub time: Duration, + pub outcome: Result, +} + +impl TestReport { + pub fn new( + package: &PackageVersionWithPackage, + runner_id: String, + runner_version: String, + compiler_backend: String, + time: Duration, + outcome: Result, + ) -> Self { + Self { + package_namespace: match &package.package.namespace { + Some(ns) => ns.clone(), + None => String::from("unknown_namespace"), + }, + package_name: package.package.package_name.clone(), + package_version: package.version.clone(), + runner_id, + runner_version, + compiler_backend, + time, + outcome, + } + } + + pub fn to_test(&self, _config: Arc) -> bool { + // In time we will have more checks to add here. + true + } +} diff --git a/tests/wasmer-argus/src/main.rs b/tests/wasmer-argus/src/main.rs new file mode 100644 index 00000000000..18a684c52e6 --- /dev/null +++ b/tests/wasmer-argus/src/main.rs @@ -0,0 +1,18 @@ +mod argus; + +use argus::*; +use clap::Parser; +use tracing_subscriber::{fmt, prelude::*, EnvFilter}; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::registry() + .with(fmt::layer()) + .with(EnvFilter::from_default_env()) + .init(); + + let config = ArgusConfig::parse(); + + let argus = Argus::try_from(config)?; + argus.run().await +}