diff --git a/circle.yml b/circle.yml index dfb7eeb60..f70517f9f 100644 --- a/circle.yml +++ b/circle.yml @@ -214,7 +214,6 @@ jobs: executor: linux-clang-latest environment: BUILD_TYPE: Coverage - TESTS_FILTER: unittests CMAKE_OPTIONS: -DCMAKE_CXX_CLANG_TIDY=clang-tidy steps: - checkout @@ -224,27 +223,17 @@ jobs: name: "Collect coverage data" working_directory: ~/build command: | + binaries='-object bin/fizzy-unittests -object bin/fizzy-spectests -object bin/fizzy-bench' + mkdir coverage find -name '*.profraw' llvm-profdata merge *.profraw -o fizzy.profdata - llvm-cov report -use-color -instr-profile fizzy.profdata -Xdemangler llvm-cxxfilt \ - -object bin/fizzy-unittests \ - -object bin/fizzy-spectests - llvm-cov report -instr-profile fizzy.profdata -Xdemangler llvm-cxxfilt > coverage/report.txt \ - -object bin/fizzy-unittests \ - -object bin/fizzy-spectests - - llvm-cov show -format=html -instr-profile fizzy.profdata -Xdemangler llvm-cxxfilt -region-coverage-lt=100 > coverage/missing.html \ - -object bin/fizzy-unittests \ - -object bin/fizzy-spectests - llvm-cov show -format=html -instr-profile fizzy.profdata -Xdemangler llvm-cxxfilt > coverage/full.html \ - -object bin/fizzy-unittests \ - -object bin/fizzy-spectests - - llvm-cov export -instr-profile fizzy.profdata -format=lcov > fizzy.lcov \ - -object bin/fizzy-unittests \ - -object bin/fizzy-spectests + llvm-cov report -use-color -instr-profile fizzy.profdata -Xdemangler llvm-cxxfilt $binaries + llvm-cov report -instr-profile fizzy.profdata -Xdemangler llvm-cxxfilt $binaries > coverage/report.txt + llvm-cov show -format=html -instr-profile fizzy.profdata -Xdemangler llvm-cxxfilt -region-coverage-lt=100 $binaries > coverage/missing.html + llvm-cov show -format=html -instr-profile fizzy.profdata -Xdemangler llvm-cxxfilt $binaries > coverage/full.html + llvm-cov export -instr-profile fizzy.profdata -format=lcov $binaries > fizzy.lcov genhtml fizzy.lcov -o coverage -t Fizzy - store_artifacts: path: ~/build/coverage diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f1966ea74..e1dd301c3 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -19,6 +19,7 @@ add_subdirectory(bench) add_subdirectory(bench_internal) add_subdirectory(spectests) add_subdirectory(unittests) +add_subdirectory(smoketests) if(FIZZY_FUZZING) add_subdirectory(fuzzer) diff --git a/test/bench/bench.cpp b/test/bench/bench.cpp index 6d72939fe..9e4a66d93 100644 --- a/test/bench/bench.cpp +++ b/test/bench/bench.cpp @@ -49,7 +49,7 @@ void benchmark_parse( // Pre-run for validation if (!engine->parse(wasm_binary)) - return state.SkipWithError("Parsing failed"); + state.SkipWithError("Parsing failed"); const auto input_size = wasm_binary.size(); auto num_bytes_parsed = uint64_t{0}; @@ -88,29 +88,35 @@ struct ExecutionBenchmarkCase /// Performs test run of the benchmark case and checks results produces by the engine /// against the expected values. -void validate_benchmark_case(benchmark::State& state, fizzy::test::WasmEngine& engine, +bool validate_benchmark_case(benchmark::State& state, fizzy::test::WasmEngine& engine, const ExecutionBenchmarkCase& benchmark_case) { if (!engine.instantiate(*benchmark_case.wasm_binary)) - return state.SkipWithError("Instantiation failed"); + { + state.SkipWithError("Instantiation failed"); + return false; + } const auto func_ref = engine.find_function(benchmark_case.func_name, benchmark_case.func_sig); if (!func_ref) { - return state.SkipWithError( - ("Function \"" + benchmark_case.func_name + "\" not found").c_str()); + state.SkipWithError(("Function \"" + benchmark_case.func_name + "\" not found").c_str()); + return false; } - if (!benchmark_case.memory.empty()) + if (!benchmark_case.memory.empty() && !engine.init_memory(benchmark_case.memory)) { - if (!engine.init_memory(benchmark_case.memory)) - return state.SkipWithError("Memory initialization failed"); + state.SkipWithError("Memory initialization failed"); + return false; } // Execute once and check results against expectations. const auto result = engine.execute(*func_ref, benchmark_case.func_args); if (result.trapped) - return state.SkipWithError("Trapped"); + { + state.SkipWithError("Trapped"); + return false; + } if (benchmark_case.expected_result) { @@ -118,43 +124,59 @@ void validate_benchmark_case(benchmark::State& state, fizzy::test::WasmEngine& e { const auto error_msg = "Missing result value, expected: " + std::to_string(*benchmark_case.expected_result); - return state.SkipWithError(error_msg.c_str()); + state.SkipWithError(error_msg.c_str()); + return false; } else if (*result.value != *benchmark_case.expected_result) { const auto error_msg = "Incorrect result value, expected: " + std::to_string(*benchmark_case.expected_result) + ", got: " + std::to_string(*result.value); - return state.SkipWithError(error_msg.c_str()); + state.SkipWithError(error_msg.c_str()); + return false; } } else if (result.value) { - return state.SkipWithError( - ("Unexpected result value: " + std::to_string(*result.value)).c_str()); + state.SkipWithError(("Unexpected result value: " + std::to_string(*result.value)).c_str()); + return false; } const auto memory = engine.get_memory(); if (memory.size() < benchmark_case.expected_memory.size()) - return state.SkipWithError("Result memory is shorter than expected"); + { + state.SkipWithError("Result memory is shorter than expected"); + return false; + } // Compare _beginning_ segment of the memory with expected. // Specifying expected full memory pages is impractical. if (!std::equal(std::begin(benchmark_case.expected_memory), std::end(benchmark_case.expected_memory), std::begin(memory))) - return state.SkipWithError("Incorrect result memory"); + { + state.SkipWithError("Incorrect result memory"); + return false; + } + + return true; } void benchmark_execute( benchmark::State& state, EngineCreateFn create_fn, const ExecutionBenchmarkCase& benchmark_case) { - const auto engine = create_fn(); + const auto has_memory = !benchmark_case.memory.empty(); - validate_benchmark_case(state, *engine, benchmark_case); + const auto engine = create_fn(); + const bool ok = validate_benchmark_case(state, *engine, benchmark_case); - const auto has_memory = !benchmark_case.memory.empty(); - engine->instantiate(*benchmark_case.wasm_binary); - const auto func_ref = engine->find_function(benchmark_case.func_name, benchmark_case.func_sig); + std::optional func_ref; + if (ok) + { + engine->instantiate(*benchmark_case.wasm_binary); + func_ref = engine->find_function(benchmark_case.func_name, benchmark_case.func_sig); + } + // The loop body will not be executed if validation error reported with state.SkipWithError(). + // However, due to a bug in libbenchmark, the loop must be always reached. for ([[maybe_unused]] auto _ : state) { state.PauseTiming(); diff --git a/test/smoketests/CMakeLists.txt b/test/smoketests/CMakeLists.txt new file mode 100644 index 000000000..102a69ea1 --- /dev/null +++ b/test/smoketests/CMakeLists.txt @@ -0,0 +1,5 @@ +# Fizzy: A fast WebAssembly interpreter +# Copyright 2020 The Fizzy Authors. +# SPDX-License-Identifier: Apache-2.0 + +add_subdirectory(benchmarks) diff --git a/test/smoketests/benchmarks/CMakeLists.txt b/test/smoketests/benchmarks/CMakeLists.txt new file mode 100644 index 000000000..155d4705a --- /dev/null +++ b/test/smoketests/benchmarks/CMakeLists.txt @@ -0,0 +1,38 @@ +# Fizzy: A fast WebAssembly interpreter +# Copyright 2020 The Fizzy Authors. +# SPDX-License-Identifier: Apache-2.0 + +add_test( + NAME fizzy/smoketests/bench/benchmarks + COMMAND fizzy-bench ${CMAKE_CURRENT_LIST_DIR} --benchmark_min_time=0.01 +) + +add_test( + NAME fizzy/smoketests/bench/cli-missing-dir-arg + COMMAND fizzy-bench +) +set_tests_properties( + fizzy/smoketests/bench/cli-missing-dir-arg + PROPERTIES + PASS_REGULAR_EXPRESSION "Missing DIR argument" +) + +add_test( + NAME fizzy/smoketests/bench/cli-invalid-arg + COMMAND fizzy-bench ${CMAKE_CURRENT_LIST_DIR} --nonexisting_argument +) +set_tests_properties( + fizzy/smoketests/bench/cli-invalid-arg + PROPERTIES + PASS_REGULAR_EXPRESSION "error: unrecognized command-line flag: --nonexisting_argument" +) + + +# Dump coverage data to distinct files (otherwise file will be overwritten). +set_tests_properties( + fizzy/smoketests/bench/benchmarks + fizzy/smoketests/bench/cli-missing-dir-arg + fizzy/smoketests/bench/cli-invalid-arg + PROPERTIES + ENVIRONMENT LLVM_PROFILE_FILE=${CMAKE_BINARY_DIR}/bench-%p.profraw +) diff --git a/test/smoketests/benchmarks/arith.inputs b/test/smoketests/benchmarks/arith.inputs new file mode 100644 index 000000000..b4c5e0800 --- /dev/null +++ b/test/smoketests/benchmarks/arith.inputs @@ -0,0 +1,50 @@ +addition +add +ii:i +1 2 + +3 + +division_by_zero +div +ii:i +1 0 + +0 + +memory_initialization_failure +add +ii:i +0 0 +fe +0 + +unexcepted_result +div +ii:i +1 1 + + + +expected_memory_shorter +add +ii:i +0 0 + +0 +00 + +missing_function +sub +ii:i +0 0 + +0 + +incorrect_result_value +add +ii:i +2 2 + +5 + diff --git a/test/smoketests/benchmarks/arith.wasm b/test/smoketests/benchmarks/arith.wasm new file mode 100644 index 000000000..6ea35a964 Binary files /dev/null and b/test/smoketests/benchmarks/arith.wasm differ diff --git a/test/smoketests/benchmarks/arith.wat b/test/smoketests/benchmarks/arith.wat new file mode 100644 index 000000000..900c0318d --- /dev/null +++ b/test/smoketests/benchmarks/arith.wat @@ -0,0 +1,12 @@ +(module + (func (export "add") (param i32 i32) (result i32) + local.get 0 + local.get 1 + i32.add + ) + (func (export "div") (param i32 i32) (result i32) + local.get 0 + local.get 1 + i32.div_u + ) +) diff --git a/test/smoketests/benchmarks/invalid.inputs b/test/smoketests/benchmarks/invalid.inputs new file mode 100644 index 000000000..86cb86351 --- /dev/null +++ b/test/smoketests/benchmarks/invalid.inputs @@ -0,0 +1,7 @@ +add +add +ii:i +1 2 + + + diff --git a/test/smoketests/benchmarks/invalid.wasm b/test/smoketests/benchmarks/invalid.wasm new file mode 100644 index 000000000..a6dc24136 Binary files /dev/null and b/test/smoketests/benchmarks/invalid.wasm differ diff --git a/test/smoketests/benchmarks/invalid.wat b/test/smoketests/benchmarks/invalid.wat new file mode 100644 index 000000000..8bdc67c82 --- /dev/null +++ b/test/smoketests/benchmarks/invalid.wat @@ -0,0 +1,6 @@ +(module + (func (export "add") (param i32 i32) (result i32) + local.get 0 + i32.add + ) +) diff --git a/test/smoketests/benchmarks/malformed/malformed.inputs b/test/smoketests/benchmarks/malformed/malformed.inputs new file mode 100644 index 000000000..064451778 --- /dev/null +++ b/test/smoketests/benchmarks/malformed/malformed.inputs @@ -0,0 +1,7 @@ +fake +fake_func +: + + + + diff --git a/test/smoketests/benchmarks/malformed/malformed.wasm b/test/smoketests/benchmarks/malformed/malformed.wasm new file mode 100644 index 000000000..4f9442719 Binary files /dev/null and b/test/smoketests/benchmarks/malformed/malformed.wasm differ diff --git a/test/smoketests/benchmarks/memory.inputs b/test/smoketests/benchmarks/memory.inputs new file mode 100644 index 000000000..466dd630a --- /dev/null +++ b/test/smoketests/benchmarks/memory.inputs @@ -0,0 +1,24 @@ +increase_value +inc +i: +0 +ff000000 + +00010000 + + +wrong_expected_memory +inc +i: +0 +ff000000 + +ffffffff + +missing_result_value +inc +i: +0 + +0 + diff --git a/test/smoketests/benchmarks/memory.wasm b/test/smoketests/benchmarks/memory.wasm new file mode 100644 index 000000000..aa20ec8bf Binary files /dev/null and b/test/smoketests/benchmarks/memory.wasm differ diff --git a/test/smoketests/benchmarks/memory.wat b/test/smoketests/benchmarks/memory.wat new file mode 100644 index 000000000..ab4ca9095 --- /dev/null +++ b/test/smoketests/benchmarks/memory.wat @@ -0,0 +1,11 @@ +(module + (memory 1 1) + (func (export "inc") (param i32) + local.get 0 + local.get 0 + i32.load + i32.const 1 + i32.add + i32.store + ) +) diff --git a/test/smoketests/benchmarks/missing_imported_function.inputs b/test/smoketests/benchmarks/missing_imported_function.inputs new file mode 100644 index 000000000..476b57fde --- /dev/null +++ b/test/smoketests/benchmarks/missing_imported_function.inputs @@ -0,0 +1,7 @@ +main +main +i:i +0 + +0 + diff --git a/test/smoketests/benchmarks/missing_imported_function.wasm b/test/smoketests/benchmarks/missing_imported_function.wasm new file mode 100644 index 000000000..5b59bd19d Binary files /dev/null and b/test/smoketests/benchmarks/missing_imported_function.wasm differ diff --git a/test/smoketests/benchmarks/missing_imported_function.wat b/test/smoketests/benchmarks/missing_imported_function.wat new file mode 100644 index 000000000..2711b9d61 --- /dev/null +++ b/test/smoketests/benchmarks/missing_imported_function.wat @@ -0,0 +1,7 @@ +(module + (func $imported (import "env" "imported") (param i32) (result i32)) + (func (export "main") (param i32) (result i32) + local.get 0 + call $imported + ) +) diff --git a/test/utils/fizzy_engine.cpp b/test/utils/fizzy_engine.cpp index 6a41c2974..191b42cc0 100644 --- a/test/utils/fizzy_engine.cpp +++ b/test/utils/fizzy_engine.cpp @@ -135,7 +135,6 @@ WasmEngine::Result FizzyEngine::execute( { const auto [trapped, result_stack] = fizzy::execute(*m_instance, static_cast(func_ref), args); - assert(result_stack.size() <= 1); return {trapped, !result_stack.empty() ? result_stack.back() : std::optional{}}; } } // namespace fizzy::test