Skip to content

Commit

Permalink
test: Improve spectest runner output
Browse files Browse the repository at this point in the history
  • Loading branch information
@gumb0
gumb0 committed Aug 4, 2020
1 parent 0728606 commit 4e05bc0
Showing 1 changed file with 48 additions and 20 deletions.
68 changes: 48 additions & 20 deletions test/spectests/spectests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ fizzy::bytes load_wasm_file(const fs::path& json_file_path, std::string_view fil
struct test_settings
{
bool skip_validation = false;
bool show_passed = false;
bool show_failed = true;
bool show_skipped = false;
};

struct test_results
Expand Down Expand Up @@ -96,14 +99,12 @@ class test_runner

for (const auto& cmd : j.at("commands"))
{
const auto type = cmd.at("type").get<std::string>();
m_line = cmd.at("line").get<int>();
m_type = cmd.at("type").get<std::string>();

log_no_newline("Line " + std::to_string(cmd.at("line").get<int>()) + ": " + type + " ");

if (type == "module")
if (m_type == "module")
{
const auto filename = cmd.at("filename").get<std::string>();
log_no_newline("Instantiating " + filename + " ");

const std::string name =
(cmd.find("name") != cmd.end() ? cmd.at("name").get<std::string>() :
Expand Down Expand Up @@ -151,7 +152,7 @@ class test_runner
}
pass();
}
else if (type == "register")
else if (m_type == "register")
{
const auto module_name =
(cmd.find("name") != cmd.end() ? cmd["name"] : UnnamedModule);
Expand Down Expand Up @@ -185,7 +186,7 @@ class test_runner
m_registered_names[registered_name] = module_name;
pass();
}
else if (type == "assert_return" || type == "action")
else if (m_type == "assert_return" || m_type == "action")
{
const auto& action = cmd.at("action");
const auto action_type = action.at("type").get<std::string>();
Expand Down Expand Up @@ -238,7 +239,7 @@ class test_runner
else
skip("Unsupported action type '" + action_type + "'");
}
else if (type == "assert_trap" || type == "assert_exhaustion")
else if (m_type == "assert_trap" || m_type == "assert_exhaustion")
{
const auto& action = cmd.at("action");
const auto action_type = action.at("type").get<std::string>();
Expand All @@ -260,11 +261,11 @@ class test_runner

pass();
}
else if (type == "assert_invalid" || type == "assert_malformed")
else if (m_type == "assert_invalid" || m_type == "assert_malformed")
{
// NOTE: assert_malformed should result in a parser error and
// assert_invalid should result in a validation error
if (type == "assert_invalid" && m_settings.skip_validation)
if (m_type == "assert_invalid" && m_settings.skip_validation)
{
skip("Validation tests disabled.");
continue;
Expand All @@ -285,15 +286,15 @@ class test_runner
}
catch (fizzy::parser_error const& ex)
{
if (type == "assert_malformed")
if (m_type == "assert_malformed")
pass();
else
fail(std::string{"Unexpected parser error: "} + ex.what());
continue;
}
catch (fizzy::validation_error const& ex)
{
if (type == "assert_invalid")
if (m_type == "assert_invalid")
pass();
else
fail(std::string{"Unexpected validation error: "} + ex.what());
Expand All @@ -303,7 +304,7 @@ class test_runner
fail("Invalid module parsed successfully. Expected error: " +
cmd.at("text").get<std::string>());
}
else if (type == "assert_unlinkable" || type == "assert_uninstantiable")
else if (m_type == "assert_unlinkable" || m_type == "assert_uninstantiable")
{
// NOTE: assert_uninstantiable should result in a start function trap
// assert_unlinkable checks all other instantiation failures
Expand All @@ -324,7 +325,7 @@ class test_runner
auto [imports, error] = create_imports(module);
if (!error.empty())
{
if (type == "assert_unlinkable")
if (m_type == "assert_unlinkable")
pass();
else
fail(error);
Expand All @@ -349,14 +350,14 @@ class test_runner
{
if (ex.what() == std::string{"start function failed to execute"})
{
if (type == "assert_uninstantiable")
if (m_type == "assert_uninstantiable")
pass();
else
fail(std::string{"Instantiation failed with error: "} + ex.what());
}
else
{
if (type == "assert_uninstantiable")
if (m_type == "assert_uninstantiable")
fail(std::string{"Instantiation failed with error: "} + ex.what());
else
pass();
Expand All @@ -371,6 +372,9 @@ class test_runner
skip("Unsupported command type");
}

log("");
if (!m_result_summary.empty())
log_no_newline(m_result_summary);
log(std::to_string(m_results.passed + m_results.failed + m_results.skipped) +
" tests ran from " + path.filename().string() + ".\n PASSED " +
std::to_string(m_results.passed) + ", FAILED " + std::to_string(m_results.failed) +
Expand Down Expand Up @@ -531,30 +535,48 @@ class test_runner
void pass()
{
++m_results.passed;
std::cout << "PASSED\n";
if (m_settings.show_passed)
add_to_result_summary("PASSED");
std::cout << ".";
}

void fail(std::string_view message)
{
++m_results.failed;
std::cout << "FAILED " << message << "\n";
if (m_settings.show_failed)
add_to_result_summary("FAILED " + std::string{message});
std::cout << "F";
}

void skip(std::string_view message)
{
++m_results.skipped;
std::cout << "SKIPPED " << message << "\n";
if (m_settings.show_skipped)
add_to_result_summary("SKIPPED " + std::string{message});
std::cout << "S";
}

static void log(std::string_view message) { std::cout << message << "\n"; }

static void log_no_newline(std::string_view message) { std::cout << message << std::flush; }
void log_no_newline(std::string_view message) { std::cout << message; }

void add_to_result_summary(std::string_view message)
{
assert(!m_type.empty() && m_line != 0);
m_result_summary +=
"Line " + std::to_string(m_line) + ": " + m_type + " " + std::string(message) + "\n";
m_line = 0;
m_type.clear();
}

test_settings m_settings;
std::unordered_map<std::string, std::unique_ptr<fizzy::Instance>> m_instances;
std::unordered_map<std::string, std::string> m_registered_names;
std::string m_last_module_name;
test_results m_results;
std::string m_result_summary;
int m_line = 0;
std::string m_type;
};

bool run_tests_from_dir(const fs::path& path, const test_settings& settings)
Expand Down Expand Up @@ -600,6 +622,12 @@ int main(int argc, char** argv)
{
if (argv[i] == std::string{"--skip-validation"})
settings.skip_validation = true;
else if (argv[i] == std::string{"--hide-failed"})
settings.show_failed = false;
else if (argv[i] == std::string{"--show-passed"})
settings.show_passed = true;
else if (argv[i] == std::string{"--show-skipped"})
settings.show_skipped = true;
else
{
std::cerr << "Unknown argument: " << argv[i] << "\n";
Expand Down

0 comments on commit 4e05bc0

Please sign in to comment.