test: add vm-level fuzzing (#5462)

Finally got to publishing fuzzing infra we have been using in ad-hoc ways before. This just runs the contracts to make sure that VM doesn't crash. The code is setup to easily compare execution with different VM Kinds. 

Note that at the moment fuzzing *does not* generally exercise host function calls. I need to finish bytecodealliance/wasm-tools#286 for that 😅

Cargo.toml

@@ -9,6 +9,7 @@ members = [
     "core/metrics",
     "runtime/near-vm-logic",
     "runtime/near-vm-runner",
+    "runtime/near-vm-runner/fuzz",
     "runtime/near-vm-runner-standalone",
     "runtime/runtime",
     "runtime/runtime-params-estimator",

nightly/fuzzing.txt

@@ -1 +1,2 @@
-pytest --skip-build --timeout=2h runtime/fuzz.py
+pytest --skip-build --timeout=2h runtime/fuzz_runtime.py
+pytest --skip-build --timeout=2h runtime/fuzz_wasm_vm.py

pytest/tests/runtime/fuzz.py → pytest/lib/cargo_fuzz.py

@@ -1,10 +1,26 @@
-#!/usr/bin/env python3
+# Running `cargo-fuzz` based fuzzers.
 import os
-import subprocess
 import sys
+import subprocess
+
+
+def run(dir: str, fuzz_target: str) -> int:
+    args = ('cargo', 'fuzz', 'run', fuzz_target, '--', '-len_control=0'
+            '-prefer_small=0', '-max_len=4000000', '-rss_limit_mb=10240')
+    os.environ['RUSTC_BOOTSTRAP'] = '1'
+    try:
+        # libfuzzer has a -max_total_time flag however it does not measure time
+        # compilation takes.  Because of that, rather than using that option
+        # we’re handling timeout over the entire command ourselves.
+        return subprocess.call(args,
+                               cwd=os.path.join('..', dir),
+                               timeout=_get_timeout())
+    except subprocess.TimeoutExpired:
+        print('No failures found.')
+        return 0
 
 
-def get_timeout():
+def _get_timeout():
     timeout = os.environ.get('NAYDUCK_TIMEOUT')
     if timeout:
         try:
@@ -22,23 +38,3 @@ def get_timeout():
         'Test will run until failure is found or it’s interrupted.',
         file=sys.stderr)
     return None
-
-
-def main() -> int:
-    args = ('cargo', 'fuzz', 'run', 'runtime-fuzzer', '--', '-len_control=0'
-            '-prefer_small=0', '-max_len=4000000', '-rss_limit_mb=10240')
-    os.environ['RUSTC_BOOTSTRAP'] = '1'
-    try:
-        # libfuzzer has a -max_total_time flag however it does not measure time
-        # compilation takes.  Because of that, rather than using that option
-        # we’re handling timeout over the entire command ourselves.
-        return subprocess.call(args,
-                               cwd='../test-utils/runtime-tester/fuzz',
-                               timeout=get_timeout())
-    except subprocess.TimeoutExpired:
-        print('No failures found.')
-        return 0
-
-
-if __name__ == '__main__':
-    sys.exit(main())

pytest/tests/runtime/fuzz_runtime.py

@@ -0,0 +1,10 @@
+import os
+import subprocess
+import sys
+
+sys.path.append('lib')
+
+import cargo_fuzz
+
+if __name__ == '__main__':
+    sys.exit(cargo_fuzz.run('test-utils/runtime-tester/fuzz', 'runtime-fuzzer'))

pytest/tests/runtime/fuzz_wasm_vm.py

@@ -0,0 +1,10 @@
+import os
+import subprocess
+import sys
+
+sys.path.append('lib')
+
+import cargo_fuzz
+
+if __name__ == '__main__':
+    sys.exit(cargo_fuzz.run('runtime/near-vm-runner/fuzz', 'runner'))

runtime/near-vm-runner/README.md

@@ -32,13 +32,19 @@ to programmatically drive the runner for benchmarking or ad-hoc investigations.
 
 There's a bunch of unit-tests in this crate. You can run them with
 
-```bash
-cargo t -p near-vm-runner --features wasmer0_vm,wasmer2_vm,wasmtime_vm
+```console
+$ cargo t -p near-vm-runner --features wasmer0_vm,wasmer2_vm,wasmtime_vm
 ```
 
 The tests use either a short wasm snippets specified inline, or a couple of
 larger test contracts from the `near-test-contracts` crate.
 
+We also have fuzzing setup:
+
+```console
+$ cd runtime/near-vm-runner && RUSTC_BOOTSTRAP=1 cargo fuzz run runner
+```
+
 ## Profiling
 
 `tracing` crate is used to collect Rust code profile data via manual instrumentation.

runtime/near-vm-runner/fuzz/.gitignore

@@ -0,0 +1,2 @@
+corpus
+artifacts

runtime/near-vm-runner/fuzz/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "near-vm-runner-fuzz"
+version = "0.0.0"
+publish = false
+edition = "2021"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+libfuzzer-sys = "0.4"
+wasm-smith = "0.8"
+wasmprinter = "0.2"
+arbitrary = "1"
+
+near-vm-runner = { path = ".." }
+near-vm-logic = { path = "../../near-vm-logic", default-features = false, features = [] }
+near-primitives = { path = "../../../core/primitives" }
+
+
+[[bin]]
+name = "runner"
+path = "fuzz_targets/runner.rs"
+test = false
+doc = false

runtime/near-vm-runner/fuzz/fuzz_targets/runner.rs

@@ -0,0 +1,108 @@
+#![no_main]
+
+use arbitrary::Arbitrary;
+use core::fmt;
+use near_primitives::contract::ContractCode;
+use near_primitives::runtime::fees::RuntimeFeesConfig;
+use near_primitives::version::PROTOCOL_VERSION;
+use near_vm_logic::mocks::mock_external::MockedExternal;
+use near_vm_logic::{VMConfig, VMContext, VMOutcome};
+use near_vm_runner::internal::wasmparser::{Export, ExternalKind, Parser, Payload, TypeDef};
+use near_vm_runner::internal::VMKind;
+use near_vm_runner::VMError;
+
+libfuzzer_sys::fuzz_target!(|module: ArbitraryModule| {
+    let code = ContractCode::new(module.0.to_bytes(), None);
+    let (_outcome, _err) = run_fuzz(&code, VMKind::Wasmer0);
+});
+
+fn run_fuzz(code: &ContractCode, vm_kind: VMKind) -> (Option<VMOutcome>, Option<VMError>) {
+    let mut fake_external = MockedExternal::new();
+    let mut context = create_context(vec![]);
+    context.prepaid_gas = 10u64.pow(14);
+    let config = VMConfig::test();
+    let fees = RuntimeFeesConfig::test();
+
+    let promise_results = vec![];
+
+    let method_name = find_entry_point(code).unwrap_or_else(|| "main".to_string());
+    vm_kind.runtime().unwrap().run(
+        &code,
+        &method_name,
+        &mut fake_external,
+        context,
+        &config,
+        &fees,
+        &promise_results,
+        PROTOCOL_VERSION,
+        None,
+    )
+}
+
+/// Finds a no-parameter exported function, something like `(func (export "entry-point"))`.
+fn find_entry_point(contract: &ContractCode) -> Option<String> {
+    let mut tys = Vec::new();
+    let mut fns = Vec::new();
+    for payload in Parser::default().parse_all(contract.code()) {
+        match payload {
+            Ok(Payload::FunctionSection(rdr)) => fns.extend(rdr),
+            Ok(Payload::TypeSection(rdr)) => tys.extend(rdr),
+            Ok(Payload::ExportSection(rdr)) => {
+                for export in rdr {
+                    if let Ok(Export { field, kind: ExternalKind::Function, index }) = export {
+                        if let Some(&Ok(ty_index)) = fns.get(index as usize) {
+                            if let Some(Ok(TypeDef::Func(func_type))) = tys.get(ty_index as usize) {
+                                if func_type.params.is_empty() && func_type.returns.is_empty() {
+                                    return Some(field.to_string());
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            _ => (),
+        }
+    }
+    None
+}
+
+fn create_context(input: Vec<u8>) -> VMContext {
+    VMContext {
+        current_account_id: "alice".parse().unwrap(),
+        signer_account_id: "bob".parse().unwrap(),
+        signer_account_pk: vec![0, 1, 2, 3, 4],
+        predecessor_account_id: "carol".parse().unwrap(),
+        input,
+        block_index: 10,
+        block_timestamp: 42,
+        epoch_height: 1,
+        account_balance: 2u128,
+        account_locked_balance: 0,
+        storage_usage: 12,
+        attached_deposit: 2u128,
+        prepaid_gas: 10_u64.pow(14),
+        random_seed: vec![0, 1, 2],
+        view_config: None,
+        output_data_receivers: vec![],
+    }
+}
+
+/// Silly wrapper to get more useful Debug.
+struct ArbitraryModule(wasm_smith::Module);
+
+impl<'a> Arbitrary<'a> for ArbitraryModule {
+    fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result<Self> {
+        wasm_smith::Module::arbitrary(u).map(ArbitraryModule)
+    }
+}
+
+impl fmt::Debug for ArbitraryModule {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let bytes = self.0.to_bytes();
+        write!(f, "{:?}", bytes)?;
+        if let Ok(wat) = wasmprinter::print_bytes(&bytes) {
+            write!(f, "\n{}", wat)?;
+        }
+        Ok(())
+    }
+}

runtime/near-vm-runner/src/lib.rs

@@ -33,4 +33,5 @@ pub use runner::{run, VM};
 #[doc(hidden)]
 pub mod internal {
     pub use crate::vm_kind::VMKind;
+    pub use wasmparser;
 }