Merge #1382

1382: Test Refactor r=syrusakbary a=syrusakbary

# Description

This PR is the continuation of #1380.
It refactors our testing infrastructure based on various points:
* There is no longer a "default compiler" on tests
* Tests are automatically generated for: emscripten and wasi
* It accelerates testing into multiple threads when possible (with the exception of the llvm backend)
* It automatically detects the backends available in the host

Summary of changes:
* [x] Removed all extra logic for test creation in emscripten test generator
* [x] Removed all extra logic for test creation in the wasi test generator
* [x] Divided wasmer wast test in different files (under `tests/custom/`)
* [x] Refactored trap asserts using the wast format
* [x] Improved the build script by adding emscripten and wasi test generators (separated from build)
* [x] Improved WASI errors to be Rust error types (by deriving from `thiserror::Error`)
* [x] Fixed toolchain error creation
* [x] Removed leaking of testing logic from generators into build
* [x] Refactored import tests to be compatible with multiple backends
* [x] Improved wasmer binary to work even when no backends are set (useful for testing without any backend available).
* [x] Removed assumption that Cranelift will be the default in the `wasmer` binary
* [x] Migrated middleware

# Review

- [x] Add a short description of the the change to the CHANGELOG.md file


Co-authored-by: Syrus <[email protected]>
Co-authored-by: Syrus Akbary <[email protected]>

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## **[Unreleased]**
 
+- [#1382](https://github.com/wasmerio/wasmer/pull/1382) Refactored test infranstructure (part 2)
+- [#1380](https://github.com/wasmerio/wasmer/pull/1380) Refactored test infranstructure (part 1)
 - [#1357](https://github.com/wasmerio/wasmer/pull/1357) Refactored bin commands into separate files
 - [#1331](https://github.com/wasmerio/wasmer/pull/1331) Implement the `record` type and instrutions for WIT
 - [#1345](https://github.com/wasmerio/wasmer/pull/1345) Adding ARM testing in Azure Pipelines

Cargo.toml

@@ -18,6 +18,7 @@ include = [
     "/README.md",
     "rustfmt.toml"
 ]
+autoexamples = false
 
 [dependencies]
 atty = "0.2"
@@ -27,16 +28,20 @@ fern = { version = "0.5", features = ["colored"], optional = true }
 log = "0.4"
 structopt = "0.3"
 wabt = { version = "0.9.1", optional = true }
-wasmer = { path = "lib/api" }
-wasmer-clif-backend = { path = "lib/clif-backend", optional = true }
-wasmer-singlepass-backend = { path = "lib/singlepass-backend", optional = true }
+wasmer = { path = "lib/api", default-features = false }
 wasmer-middleware-common = { path = "lib/middleware-common" }
-wasmer-runtime = { path = "lib/runtime" }
+wasmer-runtime = { path = "lib/runtime", default-features = false }
 wasmer-runtime-core = { path = "lib/runtime-core" }
-wasmer-emscripten = { path = "lib/emscripten" }
+wasmer-kernel-loader = { path = "lib/kernel-loader", optional = true }
+
+# Backends
+wasmer-singlepass-backend = { path = "lib/singlepass-backend", optional = true }
+wasmer-clif-backend = { path = "lib/clif-backend", optional = true }
 wasmer-llvm-backend = { path = "lib/llvm-backend", optional = true }
+
+# Frontends
+wasmer-emscripten = { path = "lib/emscripten" }
 wasmer-wasi = { path = "lib/wasi", optional = true }
-wasmer-kernel-loader = { path = "lib/kernel-loader", optional = true }
 wasmer-wasi-experimental-io-devices = { path = "lib/wasi-experimental-io-devices", optional = true }
 
 [workspace]
@@ -76,21 +81,25 @@ rustc_version = "0.2"
 [dev-dependencies]
 anyhow = "1.0.19"
 wasmer-wast = { path = "tests/wast" }
+lazy_static = "1.4"
+# To allow parameterized tests
+test-case = "1.0.0"
 criterion = "0.3"
 glob = "0.3"
 libc = "0.2.60" # for `tests/dev-utils`'s Stdout capturing
 serde = { version = "1", features = ["derive"] } # used by the plugin example
+serde_json = "1"
 typetag = "0.1" # used by the plugin example
 wabt = "0.9.1"
 
 [features]
-default = ["fast-tests", "wasi", "backend-cranelift", "wabt"]
-"loader-kernel" = ["wasmer-kernel-loader"]
+# Don't add the backend features in default, please add them on the Makefile
+# since we might want to autoconfigure them depending on the availability on the host.
+default = ["wasi", "wabt"]
+loader-kernel = ["wasmer-kernel-loader"]
 debug = ["fern", "log/max_level_debug", "log/release_max_level_debug"]
 trace = ["fern", "log/max_level_trace", "log/release_max_level_trace"]
 docs = ["wasmer-runtime/docs"]
-# This feature will allow cargo test to run much faster
-fast-tests = []
 backend-cranelift = [
     "wasmer-clif-backend",
     "wasmer-clif-backend/generate-debug-information",
@@ -115,9 +124,10 @@ managed = ["backend-singlepass", "wasmer-runtime-core/managed"]
 
 [[example]]
 name = "plugin"
-required-features = ["wasi"]
+required-features = ["wasi", "backend-cranelift"]
 crate-type = ["bin"]
 
 [[example]]
 name = "callback"
 crate-type = ["bin"]
+required-features = ["backend-cranelift"]