implement fuzzing for statically-defined component types

This is the first part of my work to address bytecodealliance#4307.  We now generate 1000
arbitrary types and tests for those types at build time.  Each test includes a
component which imports and exports functions that take and return its
respective type.  The exported function calls the imported function, which is
implemented by the host, and the host verifies that both the host function
argument and the guest function return value match the original input value.

In terms of bytecodealliance#4307, this includes the test case generator and the static API
oracle.  I'll follow up with a dynamic API oracle in a subsequent PR.

Signed-off-by: Joel Dice <[email protected]>

Cargo.toml

@@ -60,12 +60,18 @@ wat = "1.0.46"
 once_cell = "1.9.0"
 rayon = "1.5.0"
 component-macro-test = { path = "crates/misc/component-macro-test" }
+component-test-util = { path = "crates/misc/component-test-util" }
 
 [target.'cfg(windows)'.dev-dependencies]
 windows-sys = { version = "0.36.0", features = ["Win32_System_Memory"] }
 
 [build-dependencies]
 anyhow = "1.0.19"
+arbitrary = "1.1.3"
+proc-macro2 = "1.0"
+quote = "1.0"
+wasmtime-fuzzing = { path = "crates/fuzzing" }
+rand = "0.8.5"
 
 [profile.release.build-override]
 opt-level = 0
@@ -108,7 +114,7 @@ memory-init-cow = ["wasmtime/memory-init-cow", "wasmtime-cli-flags/memory-init-c
 pooling-allocator = ["wasmtime/pooling-allocator", "wasmtime-cli-flags/pooling-allocator"]
 all-arch = ["wasmtime/all-arch"]
 posix-signals-on-macos = ["wasmtime/posix-signals-on-macos"]
-component-model = ["wasmtime/component-model", "wasmtime-wast/component-model", "wasmtime-cli-flags/component-model"]
+component-model = ["wasmtime/component-model", "wasmtime-wast/component-model", "wasmtime-cli-flags/component-model", "wasmtime-fuzzing/component-model"]
 
 # Stub feature that does nothing, for Cargo-features compatibility: the new
 # backend is the default now.

build.rs

@@ -30,6 +30,7 @@ fn main() -> anyhow::Result<()> {
             test_directory_module(out, "tests/misc_testsuite/memory64", strategy)?;
             if cfg!(feature = "component-model") {
                 test_directory_module(out, "tests/misc_testsuite/component-model", strategy)?;
+                fuzz_static_component_api_module(out)?;
             }
             Ok(())
         })?;
@@ -43,7 +44,7 @@ fn main() -> anyhow::Result<()> {
             } else {
                 println!(
                     "cargo:warning=The spec testsuite is disabled. To enable, run `git submodule \
-                 update --remote`."
+                     update --remote`."
                 );
             }
             Ok(())
@@ -185,3 +186,122 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool {
 fn platform_is_s390x() -> bool {
     env::var("CARGO_CFG_TARGET_ARCH").unwrap() == "s390x"
 }
+
+#[cfg(feature = "component-model")]
+fn fuzz_static_component_api_module(out: &mut String) -> anyhow::Result<()> {
+    use anyhow::anyhow;
+    use arbitrary::{Arbitrary, Unstructured};
+    use proc_macro2::TokenStream;
+    use quote::{format_ident, quote};
+    use rand::rngs::StdRng;
+    use rand::{Rng, SeedableRng};
+    use std::collections::HashMap;
+    use std::ops::DerefMut;
+    use wasmtime_fuzzing::generators::component_types::{
+        self, TestCase, Type, EXPORT_FUNCTION, IMPORT_FUNCTION,
+    };
+
+    let seed = if let Ok(seed) = env::var("WASMTIME_FUZZ_SEED") {
+        seed.parse::<u64>()
+            .with_context(|| anyhow!("expected u64 in WASMTIME_FUZZ_SEED"))?
+    } else {
+        StdRng::from_entropy().gen()
+    };
+
+    let mut rng = StdRng::seed_from_u64(seed);
+
+    const TEST_CASE_COUNT: usize = 1000;
+
+    let mut tests = TokenStream::new();
+
+    for index in 0..TEST_CASE_COUNT {
+        let mut bytes = vec![0u8; rng.gen_range(1000..2000)];
+        rng.fill(bytes.deref_mut());
+
+        let TestCase {
+            ty,
+            value,
+            component,
+        } = TestCase::arbitrary(&mut Unstructured::new(&bytes))?;
+
+        let mut declarations = TokenStream::new();
+        let mut names = HashMap::new();
+        let rust_type = component_types::rust_type(&ty, &mut 0, &mut declarations, &mut names);
+        let input = component_types::rust_value(&ty, &value, &names);
+
+        let assertion = match ty {
+            Type::Float32 | Type::Float64 => {
+                quote!(input == output || (input.is_nan() && output.is_nan()))
+            }
+            _ => quote!(input == output),
+        };
+
+        let test = format_ident!("test{index}");
+
+        let test = quote! {
+            #[test]
+            fn #test() -> Result<()> {
+                #declarations
+
+                let input = #input;
+
+                let engine = component_test_util::engine();
+                let mut store = Store::new(&engine, ());
+                let component = Component::new(&engine, #component)?;
+                let mut linker = Linker::new(&engine);
+                let state = Arc::new(State::default());
+
+                linker.root().func_wrap(#IMPORT_FUNCTION, {
+                    let input = input.clone();
+                    let state = state.clone();
+
+                    move |output: #rust_type| -> Result<#rust_type> {
+                        state.called.store(true, Ordering::Relaxed);
+                        state.matched.store(#assertion, Ordering::Relaxed);
+                        Ok(output)
+                    }})?;
+
+                let instance = linker.instantiate(&mut store, &component)?;
+
+                let output = instance
+                    .get_typed_func::<(#rust_type,), #rust_type, _>(&mut store, #EXPORT_FUNCTION)?
+                    .call_and_post_return(&mut store, (input.clone(),))?;
+
+                assert!(state.called.load(Ordering::Relaxed));
+                assert!(state.matched.load(Ordering::Relaxed));
+                assert!(#assertion);
+
+                Ok(())
+            }
+        };
+
+        tests.extend(test);
+    }
+
+    let module = quote! {
+        #[allow(unused_imports)]
+        mod fuzz_component_types {
+            use component_test_util::TypedFuncExt;
+            use anyhow::Result;
+            use wasmtime::component::{Component, ComponentType, Lift, Linker, Lower};
+            use wasmtime::Store;
+            use std::sync::atomic::{AtomicBool, Ordering};
+            use std::sync::Arc;
+
+            const _SEED_USED: u64 = #seed;
+
+            #[derive(Default)]
+            struct State {
+                called: AtomicBool,
+                matched: AtomicBool,
+            }
+
+            #tests
+
+        }
+    };
+
+    write!(out, "{module}")?;
+
+    Ok(())
+}

crates/fuzzing/Cargo.toml

@@ -9,15 +9,19 @@ license = "Apache-2.0 WITH LLVM-exception"
 
 [dependencies]
 anyhow = "1.0.22"
-arbitrary = { version = "1.1.0", features = ["derive"] }
+arbitrary = { version = "1.1.3", features = ["derive"] }
+component-test-util = { path = "../misc/component-test-util" }
 env_logger = "0.9.0"
 log = "0.4.8"
+proc-macro2 = "1.0"
+quote = "1.0"
 rayon = "1.2.1"
 target-lexicon = "0.12.3"
 tempfile = "3.3.0"
 wasmparser = "0.87.0"
 wasmprinter = "0.2.37"
 wasmtime = { path = "../wasmtime" }
+wasmtime-component-util = { path = "../component-util" }
 wasmtime-wast = { path = "../wast" }
 wasm-encoder = "0.14.0"
 wasm-smith = "0.11.2"
@@ -43,4 +47,6 @@ rand = { version = "0.8.0", features = ["small_rng"] }
 wasm-spec-interpreter = { path = "./wasm-spec-interpreter", optional = true, features = ['build-libinterpret'] }
 
 [features]
+default = ['fuzz-spec-interpreter']
 fuzz-spec-interpreter = ['wasm-spec-interpreter']
+component-model = ['wasmtime/component-model']

crates/fuzzing/src/generators.rs

@@ -19,6 +19,9 @@ mod single_inst_module;
 mod spec_test;
 pub mod table_ops;
 
+#[cfg(feature = "component-model")]
+pub mod component_types;
+
 pub use codegen_settings::CodegenSettings;
 pub use config::{Config, WasmtimeConfig};
 pub use instance_allocation_strategy::InstanceAllocationStrategy;