Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test: Improve spectest runner output #453

Merged
merged 2 commits into from
Aug 4, 2020
+68 −5
Merged

test: Improve spectest runner output #453

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
30c92cc
test: Improve spectest runner output
gumb0 Aug 3, 2020
04d3eef
test: run smoketests with verbosity configuration arguments
gumb0 Aug 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions test/smoketests/spectests/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,36 @@ set_tests_properties(
PASS_REGULAR_EXPRESSION "PASSED 25, FAILED 1, SKIPPED 4"
)

add_test(
NAME fizzy/smoketests/spectests/showpassed
COMMAND fizzy-spectests ${CMAKE_CURRENT_LIST_DIR}/default --show-passed
)
set_tests_properties(
fizzy/smoketests/spectests/showpassed
PROPERTIES
PASS_REGULAR_EXPRESSION "PASSED 26, FAILED 1, SKIPPED 3"
)

add_test(
NAME fizzy/smoketests/spectests/showskipped
COMMAND fizzy-spectests ${CMAKE_CURRENT_LIST_DIR}/default --show-skipped
)
set_tests_properties(
fizzy/smoketests/spectests/showskipped
PROPERTIES
PASS_REGULAR_EXPRESSION "PASSED 26, FAILED 1, SKIPPED 3"
)

add_test(
NAME fizzy/smoketests/spectests/hidefailed
COMMAND fizzy-spectests ${CMAKE_CURRENT_LIST_DIR}/default --hide-failed
)
set_tests_properties(
fizzy/smoketests/spectests/hidefailed
PROPERTIES
PASS_REGULAR_EXPRESSION "PASSED 26, FAILED 1, SKIPPED 3"
)

add_test(
NAME fizzy/smoketests/spectests/failures
COMMAND fizzy-spectests ${CMAKE_CURRENT_LIST_DIR}/failures
Expand Down Expand Up @@ -67,6 +97,9 @@ set_tests_properties(
set_tests_properties(
fizzy/smoketests/spectests/default
fizzy/smoketests/spectests/skipvalidation
fizzy/smoketests/spectests/showpassed
fizzy/smoketests/spectests/showskipped
fizzy/smoketests/spectests/hidefailed
fizzy/smoketests/spectests/failures
fizzy/smoketests/spectests/broken
fizzy/smoketests/spectests/cli-missing-dir-arg
Expand Down
40 changes: 35 additions & 5 deletions test/spectests/spectests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ fizzy::bytes load_wasm_file(const fs::path& json_file_path, std::string_view fil
struct test_settings
{
bool skip_validation = false;
bool show_passed = false;
bool show_failed = true;
bool show_skipped = false;
};

struct test_results
Expand Down Expand Up @@ -98,12 +101,12 @@ class test_runner
{
const auto type = cmd.at("type").get<std::string>();

log_no_newline("Line " + std::to_string(cmd.at("line").get<int>()) + ": " + type + " ");
m_current_line = cmd.at("line").get<int>();
m_current_test_type = type;

if (type == "module")
{
const auto filename = cmd.at("filename").get<std::string>();
log_no_newline("Instantiating " + filename + " ");
Copy link
Collaborator Author

@gumb0 gumb0 Aug 4, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removing this, as I don't think it was very useful, and it would complicate things a bit to keep it.
The line will read just Line 3: module PASSED (instead of Line 3: module Instantiating call.0.wasm PASSED)


const std::string name =
(cmd.find("name") != cmd.end() ? cmd.at("name").get<std::string>() :
Expand Down Expand Up @@ -371,6 +374,9 @@ class test_runner
skip("Unsupported command type");
}

log(""); // newline after dots line
if (!m_result_details.empty())
log_no_newline(m_result_details);
log(std::to_string(m_results.passed + m_results.failed + m_results.skipped) +
" tests ran from " + path.filename().string() + ".\n PASSED " +
std::to_string(m_results.passed) + ", FAILED " + std::to_string(m_results.failed) +
Expand Down Expand Up @@ -531,30 +537,48 @@ class test_runner
void pass()
{
++m_results.passed;
std::cout << "PASSED\n";
if (m_settings.show_passed)
add_to_result_details("PASSED");
log_no_newline(".");
}

void fail(std::string_view message)
{
++m_results.failed;
std::cout << "FAILED " << message << "\n";
if (m_settings.show_failed)
add_to_result_details("FAILED " + std::string{message});
log_no_newline("F");
}

void skip(std::string_view message)
{
++m_results.skipped;
std::cout << "SKIPPED " << message << "\n";
if (m_settings.show_skipped)
add_to_result_details("SKIPPED " + std::string{message});
log_no_newline("s");
}

static void log(std::string_view message) { std::cout << message << "\n"; }

static void log_no_newline(std::string_view message) { std::cout << message << std::flush; }

void add_to_result_details(std::string_view message)
{
assert(!m_current_test_type.empty() && m_current_line != 0);
m_result_details += "Line " + std::to_string(m_current_line) + ": " + m_current_test_type +
" " + std::string(message) + "\n";
m_current_line = 0;
m_current_test_type.clear();
}

test_settings m_settings;
std::unordered_map<std::string, std::unique_ptr<fizzy::Instance>> m_instances;
std::unordered_map<std::string, std::string> m_registered_names;
std::string m_last_module_name;
test_results m_results;
std::string m_result_details;
int m_current_line = 0;
std::string m_current_test_type;
};

bool run_tests_from_dir(const fs::path& path, const test_settings& settings)
Expand Down Expand Up @@ -600,6 +624,12 @@ int main(int argc, char** argv)
{
if (argv[i] == std::string{"--skip-validation"})
settings.skip_validation = true;
else if (argv[i] == std::string{"--hide-failed"})
settings.show_failed = false;
else if (argv[i] == std::string{"--show-passed"})
settings.show_passed = true;
else if (argv[i] == std::string{"--show-skipped"})
settings.show_skipped = true;
else
{
std::cerr << "Unknown argument: " << argv[i] << "\n";
Expand Down