From 085d930ff3b03ea72ba7f2025ec57c82ccc913e4 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Thu, 9 Jun 2022 23:26:29 +0000 Subject: [PATCH 1/6] Support =pass in ci.baseline.txt --- include/vcpkg-test/util.h | 10 +- include/vcpkg/base/xmlserializer.h | 1 + include/vcpkg/ci-baseline.h | 19 ++- include/vcpkg/fwd/build.h | 1 + include/vcpkg/fwd/ci-baseline.h | 1 + include/vcpkg/fwd/triplet.h | 6 + include/vcpkg/triplet.h | 1 + include/vcpkg/xunitwriter.h | 37 +++++ src/vcpkg-test/ci-baseline.cpp | 193 +++++++++++++++++--------- src/vcpkg-test/util.cpp | 6 + src/vcpkg-test/xunitwriter.cpp | 78 +++++++++++ src/vcpkg/ci-baseline.cpp | 76 ++++++++++- src/vcpkg/commands.ci.cpp | 212 ++--------------------------- src/vcpkg/install.cpp | 60 ++------ src/vcpkg/xunitwriter.cpp | 168 +++++++++++++++++++++++ 15 files changed, 540 insertions(+), 329 deletions(-) create mode 100644 include/vcpkg/fwd/triplet.h create mode 100644 include/vcpkg/xunitwriter.h create mode 100644 src/vcpkg-test/xunitwriter.cpp create mode 100644 src/vcpkg/xunitwriter.cpp diff --git a/include/vcpkg-test/util.h b/include/vcpkg-test/util.h index 1b98ad1042..4e1e50f72a 100644 --- a/include/vcpkg-test/util.h +++ b/include/vcpkg-test/util.h @@ -111,11 +111,17 @@ namespace vcpkg::Test const char* depends = "", const char* triplet = "x86-windows"); - extern const Triplet X86_WINDOWS; + extern const Triplet ARM64_WINDOWS; extern const Triplet X64_WINDOWS; - extern const Triplet X86_UWP; + extern const Triplet X64_WINDOWS_STATIC; + extern const Triplet X64_WINDOWS_STATIC_MD; + extern const Triplet X86_WINDOWS; extern const Triplet ARM_UWP; + extern const Triplet X64_UWP; + extern const Triplet X86_UWP; extern const Triplet X64_ANDROID; + extern const Triplet X64_OSX; + extern const Triplet X64_LINUX; /// /// Map of source control files by their package name. diff --git a/include/vcpkg/base/xmlserializer.h b/include/vcpkg/base/xmlserializer.h index dcb6aeac8f..a81379f105 100644 --- a/include/vcpkg/base/xmlserializer.h +++ b/include/vcpkg/base/xmlserializer.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include diff --git a/include/vcpkg/ci-baseline.h b/include/vcpkg/ci-baseline.h index fb3c4fbfaf..e2a61469bd 100644 --- a/include/vcpkg/ci-baseline.h +++ b/include/vcpkg/ci-baseline.h @@ -1,4 +1,5 @@ #pragma once +#include #include #include @@ -54,7 +55,19 @@ namespace vcpkg std::vector parse_ci_baseline(StringView text, StringView origin, ParseMessages& messages); - SortedVector parse_and_apply_ci_baseline(View lines, - ExclusionsMap& exclusions_map, - SkipFailures skip_failures); + struct CiBaselineData + { + SortedVector expected_failures; + SortedVector required_success; + }; + + CiBaselineData parse_and_apply_ci_baseline(View lines, + ExclusionsMap& exclusions_map, + SkipFailures skip_failures); + + LocalizedString format_ci_result(const PackageSpec& spec, + Build::BuildResult result, + const CiBaselineData& cidata, + StringView cifile, + bool allow_unexpected_passing); } diff --git a/include/vcpkg/fwd/build.h b/include/vcpkg/fwd/build.h index c7f74513f0..77b0114bf6 100644 --- a/include/vcpkg/fwd/build.h +++ b/include/vcpkg/fwd/build.h @@ -4,4 +4,5 @@ namespace vcpkg::Build { struct BuildInfo; struct PreBuildInfo; + enum class BuildResult; } diff --git a/include/vcpkg/fwd/ci-baseline.h b/include/vcpkg/fwd/ci-baseline.h index 3d74dfe9d3..79b80cad3f 100644 --- a/include/vcpkg/fwd/ci-baseline.h +++ b/include/vcpkg/fwd/ci-baseline.h @@ -12,6 +12,7 @@ namespace vcpkg { Skip, Fail, + Pass, }; enum class SkipFailures : bool { diff --git a/include/vcpkg/fwd/triplet.h b/include/vcpkg/fwd/triplet.h new file mode 100644 index 0000000000..3e9e1bacfa --- /dev/null +++ b/include/vcpkg/fwd/triplet.h @@ -0,0 +1,6 @@ +#pragma once + +namespace vcpkg +{ + struct Triplet; +} \ No newline at end of file diff --git a/include/vcpkg/triplet.h b/include/vcpkg/triplet.h index 575fde8b52..27ae234a2e 100644 --- a/include/vcpkg/triplet.h +++ b/include/vcpkg/triplet.h @@ -2,6 +2,7 @@ #include +#include #include #include diff --git a/include/vcpkg/xunitwriter.h b/include/vcpkg/xunitwriter.h new file mode 100644 index 0000000000..0f96938d93 --- /dev/null +++ b/include/vcpkg/xunitwriter.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include + +#include + +#include +#include +#include +#include + +namespace vcpkg +{ + struct XunitTest; + + // https://xunit.net/docs/format-xml-v2 + struct XunitWriter + { + public: + // Out of line con/destructor avoids exposing XunitTest + XunitWriter(); + ~XunitWriter(); + void add_test_results(const PackageSpec& spec, + Build::BuildResult build_result, + const ElapsedTime& elapsed_time, + const std::chrono::system_clock::time_point& start_time, + const std::string& abi_tag, + const std::vector& features); + + std::string build_xml(Triplet controlling_triplet); + + private: + std::map> m_tests; + }; +} diff --git a/src/vcpkg-test/ci-baseline.cpp b/src/vcpkg-test/ci-baseline.cpp index b589b2262c..596bf11ace 100644 --- a/src/vcpkg-test/ci-baseline.cpp +++ b/src/vcpkg-test/ci-baseline.cpp @@ -3,13 +3,17 @@ #include #include +#include #include #include #include #include +#include + using namespace vcpkg; +using vcpkg::Build::BuildResult; namespace vcpkg { @@ -20,6 +24,7 @@ namespace vcpkg { case CiBaselineState::Fail: os << "fail"; break; case CiBaselineState::Skip: os << "skip"; break; + case CiBaselineState::Pass: os << "pass"; break; default: Checks::unreachable(VCPKG_LINE_INFO); } @@ -35,7 +40,7 @@ TEST_CASE ("Parse Empty", "[ci-baseline]") CHECK(actual.empty()); } -namespace +TEST_CASE ("With Sample", "[ci-baseline]") { static constexpr StringLiteral example_input = R"(########################################################################### @@ -90,6 +95,9 @@ apr:arm64-windows=fail # Requires ATL for ARM64 to be installed in CI azure-storage-cpp:arm64-windows=fail +# Require that test ports do not cascade +vcpkg-ci-opencv:arm64-windows=pass + aubio:arm-uwp=fail aubio:x64-uwp=fail # broken when `python` is python3, https://github.com/microsoft/vcpkg/issues/18937 @@ -127,19 +135,20 @@ catch-classic:x64-windows-static-md=skip catch-classic:x86-windows = skip bill-made-up-another-skip:x64-linux=skip)"; // note no trailing newline - const auto x86_windows = Triplet::from_canonical_name("x86-windows"); - const auto x64_windows = Triplet::from_canonical_name("x64-windows"); - const auto x64_windows_static = Triplet::from_canonical_name("x64-windows-static"); - const auto x64_windows_static_md = Triplet::from_canonical_name("x64-windows-static-md"); - const auto x64_uwp = Triplet::from_canonical_name("x64-uwp"); - const auto arm64_windows = Triplet::from_canonical_name("arm64-windows"); - const auto arm_uwp = Triplet::from_canonical_name("arm-uwp"); - const auto x64_osx = Triplet::from_canonical_name("x64-osx"); - const auto x64_linux = Triplet::from_canonical_name("x64-linux"); + const auto& x86_windows = Test::X86_WINDOWS; + const auto& x64_windows = Test::X64_WINDOWS; + const auto& x64_windows_static = Test::X64_WINDOWS_STATIC; + const auto& x64_windows_static_md = Test::X64_WINDOWS_STATIC_MD; + const auto& x64_uwp = Test::X64_UWP; + const auto& arm64_windows = Test::ARM64_WINDOWS; + const auto& arm_uwp = Test::ARM_UWP; + const auto& x64_osx = Test::X64_OSX; + const auto& x64_linux = Test::X64_LINUX; std::vector expected_from_example_input{ CiBaselineLine{"apr", arm64_windows, CiBaselineState::Fail}, CiBaselineLine{"azure-storage-cpp", arm64_windows, CiBaselineState::Fail}, + CiBaselineLine{"vcpkg-ci-opencv", arm64_windows, CiBaselineState::Pass}, CiBaselineLine{"aubio", arm_uwp, CiBaselineState::Fail}, CiBaselineLine{"aubio", x64_uwp, CiBaselineState::Fail}, CiBaselineLine{"bde", x64_linux, CiBaselineState::Fail}, @@ -175,24 +184,69 @@ bill-made-up-another-skip:x64-linux=skip)"; // note no trailing newline CiBaselineLine{"catch-classic", x64_windows_static_md, CiBaselineState::Skip}, CiBaselineLine{"catch-classic", x86_windows, CiBaselineState::Skip}, CiBaselineLine{"bill-made-up-another-skip", x64_linux, CiBaselineState::Skip}}; -} // unnamed namespace -TEST_CASE ("Parse Real Prefix", "[ci-baseline]") -{ - ParseMessages m; - auto actual = parse_ci_baseline(example_input, "test", m); - CHECK(m.good()); - CHECK(expected_from_example_input == actual); -} + SECTION ("Parse Real Prefix") + { + ParseMessages m; + auto actual = parse_ci_baseline(example_input, "test", m); + CHECK(m.good()); + CHECK(expected_from_example_input.size() == actual.size()); + size_t n = std::min(expected_from_example_input.size(), actual.size()); + for (size_t i = 0; i < n; ++i) + { + CHECK(expected_from_example_input[i] == actual[i]); + } + } -TEST_CASE ("Parse Real Prefix With Trailing Newline", "[ci-baseline]") -{ - ParseMessages m; - std::string newlined_input(example_input.data(), example_input.size()); - newlined_input.push_back('\n'); - auto actual = parse_ci_baseline(newlined_input, "test", m); - CHECK(m.good()); - CHECK(expected_from_example_input == actual); + SECTION ("Parse Real Prefix With Trailing Newline") + { + ParseMessages m; + std::string newlined_input(example_input.data(), example_input.size()); + newlined_input.push_back('\n'); + auto actual = parse_ci_baseline(newlined_input, "test", m); + CHECK(m.good()); + CHECK(expected_from_example_input == actual); + } + + SECTION ("Applies Skips and Fails") + { + ExclusionsMap exclusions; + exclusions.insert(x64_uwp); // example triplet + exclusions.insert(x64_linux); // example host triplet + auto actual = parse_and_apply_ci_baseline(expected_from_example_input, exclusions, SkipFailures::No); + const SortedVector expected_expected_failures{ + PackageSpec{"aubio", x64_uwp}, + PackageSpec{"bde", x64_linux}, + PackageSpec{"blitz", x64_uwp}, + PackageSpec{"blosc", x64_uwp}, + PackageSpec{"bond", x64_uwp}, + PackageSpec{"botan", x64_uwp}, + PackageSpec{"buck-yeh-bux", x64_linux}, + PackageSpec{"buck-yeh-bux-mariadb-client", x64_linux}, + PackageSpec{"c-ares", x64_uwp}, + PackageSpec{"caf", x64_uwp}, + PackageSpec{"casclib", x64_uwp}, + }; + + CHECK(actual.expected_failures == expected_expected_failures); + CHECK(exclusions.triplets.size() == 2); + CHECK(exclusions.triplets[0].exclusions == SortedVector{"catch-classic"}); + CHECK(exclusions.triplets[1].exclusions == + SortedVector{"catch-classic", "bill-made-up-another-skip"}); + + exclusions.triplets[0].exclusions.clear(); + exclusions.triplets[1].exclusions.clear(); + + actual = parse_and_apply_ci_baseline(expected_from_example_input, exclusions, SkipFailures::Yes); + CHECK(actual.expected_failures == expected_expected_failures); + CHECK(exclusions.triplets.size() == 2); + CHECK(exclusions.triplets[0].exclusions == + SortedVector{ + "aubio", "blitz", "blosc", "bond", "botan", "c-ares", "caf", "casclib", "catch-classic"}); + CHECK(exclusions.triplets[1].exclusions == + SortedVector{ + "bde", "buck-yeh-bux", "buck-yeh-bux-mariadb-client", "catch-classic", "bill-made-up-another-skip"}); + } } static void check_error(const std::string& input, const std::string& expected_error) @@ -201,6 +255,7 @@ static void check_error(const std::string& input, const std::string& expected_er auto actual = parse_ci_baseline(input, "test", m); CHECK(actual.empty()); CHECK(m.warnings.empty()); + REQUIRE(m.error); CHECK(m.error->to_string() == expected_error); } @@ -240,16 +295,12 @@ TEST_CASE ("Parse Errors", "[ci-baseline]") on expression: port:x64-windows ^)"); - check_error("example:x64-windows = \n fail", R"(test:1:26: error: expected 'fail' or 'skip' here + check_error("example:x64-windows = \n fail", R"(test:1:26: error: expected 'fail', 'skip', or 'pass' here on expression: example:x64-windows = ^)"); - check_error("example:x64-windows = pass", R"(test:1:28: error: expected 'fail' or 'skip' here - on expression: example:x64-windows = pass - ^)"); - // note that there is 'fail' but doesn't end on a word boundary: - check_error("example:x64-windows = fails", R"(test:1:28: error: expected 'fail' or 'skip' here + check_error("example:x64-windows = fails", R"(test:1:28: error: expected 'fail', 'skip', or 'pass' here on expression: example:x64-windows = fails ^)"); @@ -270,42 +321,50 @@ TEST_CASE ("Parse Errors", "[ci-baseline]") ^)"); } -TEST_CASE ("Applies Skips and Fails", "[ci-baseline]") +TEST_CASE ("format_ci_result 1", "[ci-baseline]") { - ExclusionsMap exclusions; - exclusions.insert(x64_uwp); // example triplet - exclusions.insert(x64_linux); // example host triplet - auto actual_expected_failures = - parse_and_apply_ci_baseline(expected_from_example_input, exclusions, SkipFailures::No); - const SortedVector expected_expected_failures{ - PackageSpec{"aubio", x64_uwp}, - PackageSpec{"bde", x64_linux}, - PackageSpec{"blitz", x64_uwp}, - PackageSpec{"blosc", x64_uwp}, - PackageSpec{"bond", x64_uwp}, - PackageSpec{"botan", x64_uwp}, - PackageSpec{"buck-yeh-bux", x64_linux}, - PackageSpec{"buck-yeh-bux-mariadb-client", x64_linux}, - PackageSpec{"c-ares", x64_uwp}, - PackageSpec{"caf", x64_uwp}, - PackageSpec{"casclib", x64_uwp}, + CiBaselineData cidata; + cidata.expected_failures = { + PackageSpec{"fail", Test::X64_UWP}, + }; + cidata.required_success = { + PackageSpec{"pass", Test::X64_UWP}, }; + const char* failmsg = "REGRESSION: {0} failed with BUILD_FAILED. If expected, add {0}=fail to cifile."; + const char* cascademsg = "REGRESSION: {0} cascaded, but it is required to pass. (cifile)."; + const char* passmsg = "PASSING, REMOVE FROM FAIL LIST: {0} (cifile)."; + + SECTION ("SUCCEEDED") + { + const auto test = [&](PackageSpec s, bool allow_unexpected_passing) { + return format_ci_result(s, BuildResult::SUCCEEDED, cidata, "cifile", allow_unexpected_passing); + }; + CHECK(test({"pass", Test::X64_UWP}, true) == ""); + CHECK(test({"pass", Test::X64_UWP}, false) == ""); + CHECK(test({"fail", Test::X64_UWP}, true) == ""); + CHECK(test({"fail", Test::X64_UWP}, false) == fmt::format(passmsg, "fail:x64-uwp")); + CHECK(test({"fail", Test::ARM_UWP}, false) == ""); + CHECK(test({"neither", Test::X64_UWP}, true) == ""); + CHECK(test({"neither", Test::X64_UWP}, false) == ""); + } - CHECK(actual_expected_failures == expected_expected_failures); - CHECK(exclusions.triplets.size() == 2); - CHECK(exclusions.triplets[0].exclusions == SortedVector{"catch-classic"}); - CHECK(exclusions.triplets[1].exclusions == SortedVector{"catch-classic", "bill-made-up-another-skip"}); - - exclusions.triplets[0].exclusions.clear(); - exclusions.triplets[1].exclusions.clear(); - - actual_expected_failures = parse_and_apply_ci_baseline(expected_from_example_input, exclusions, SkipFailures::Yes); - CHECK(actual_expected_failures == expected_expected_failures); - CHECK(exclusions.triplets.size() == 2); - CHECK(exclusions.triplets[0].exclusions == - SortedVector{ - "aubio", "blitz", "blosc", "bond", "botan", "c-ares", "caf", "casclib", "catch-classic"}); - CHECK(exclusions.triplets[1].exclusions == - SortedVector{ - "bde", "buck-yeh-bux", "buck-yeh-bux-mariadb-client", "catch-classic", "bill-made-up-another-skip"}); + SECTION ("BUILD_FAILED") + { + const auto test = [&](PackageSpec s) { + return format_ci_result(s, BuildResult::BUILD_FAILED, cidata, "cifile", false); + }; + CHECK(test({"pass", Test::X64_UWP}) == fmt::format(failmsg, "pass:x64-uwp")); + CHECK(test({"fail", Test::X64_UWP}) == ""); + CHECK(test({"neither", Test::X64_UWP}) == fmt::format(failmsg, "neither:x64-uwp")); + } + + SECTION ("CASCADED_DUE_TO_MISSING_DEPENDENCIES") + { + const auto test = [&](PackageSpec s) { + return format_ci_result(s, BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES, cidata, "cifile", false); + }; + CHECK(test({"pass", Test::X64_UWP}) == fmt::format(cascademsg, "pass:x64-uwp")); + CHECK(test({"fail", Test::X64_UWP}) == ""); + CHECK(test({"neither", Test::X64_UWP}) == ""); + } } diff --git a/src/vcpkg-test/util.cpp b/src/vcpkg-test/util.cpp index c11ec23d3c..7b142439cb 100644 --- a/src/vcpkg-test/util.cpp +++ b/src/vcpkg-test/util.cpp @@ -17,11 +17,17 @@ namespace vcpkg::Test { + const Triplet ARM64_WINDOWS = Triplet::from_canonical_name("arm64-windows"); const Triplet X86_WINDOWS = Triplet::from_canonical_name("x86-windows"); const Triplet X64_WINDOWS = Triplet::from_canonical_name("x64-windows"); + const Triplet X64_WINDOWS_STATIC = Triplet::from_canonical_name("x64-windows-static"); + const Triplet X64_WINDOWS_STATIC_MD = Triplet::from_canonical_name("x64-windows-static-md"); + const Triplet X64_UWP = Triplet::from_canonical_name("x64-uwp"); const Triplet X86_UWP = Triplet::from_canonical_name("x86-uwp"); const Triplet ARM_UWP = Triplet::from_canonical_name("arm-uwp"); const Triplet X64_ANDROID = Triplet::from_canonical_name("x64-android"); + const Triplet X64_OSX = Triplet::from_canonical_name("x64-osx"); + const Triplet X64_LINUX = Triplet::from_canonical_name("x64-linux"); std::unique_ptr make_control_file( const char* name, diff --git a/src/vcpkg-test/xunitwriter.cpp b/src/vcpkg-test/xunitwriter.cpp new file mode 100644 index 0000000000..46abe41aa3 --- /dev/null +++ b/src/vcpkg-test/xunitwriter.cpp @@ -0,0 +1,78 @@ +#include + +#include +#include + +#include +#include + +using namespace vcpkg; +using Build::BuildResult; + +TEST_CASE ("Simple XunitWriter", "[xunitwriter]") +{ + XunitWriter x; + time_t time = {0}; + Triplet t = Triplet::from_canonical_name("triplet"); + PackageSpec spec("name", t); + x.add_test_results(spec, BuildResult::BUILD_FAILED, {}, std::chrono::system_clock::from_time_t(time), "", {}); + CHECK(x.build_xml(t) == R"( + + + + + + + + + + + +)"); +} + +TEST_CASE ("XunitWriter Two", "[xunitwriter]") +{ + XunitWriter x; + time_t time = {0}; + Triplet t = Triplet::from_canonical_name("triplet"); + Triplet t2 = Triplet::from_canonical_name("triplet2"); + Triplet t3 = Triplet::from_canonical_name("triplet3"); + PackageSpec spec("name", t); + PackageSpec spec2("name", t2); + PackageSpec spec3("other", t2); + x.add_test_results(spec, BuildResult::SUCCEEDED, {}, std::chrono::system_clock::from_time_t(time), "abihash", {}); + x.add_test_results( + spec2, BuildResult::POST_BUILD_CHECKS_FAILED, {}, std::chrono::system_clock::from_time_t(time), "", {}); + x.add_test_results( + spec3, BuildResult::SUCCEEDED, {}, std::chrono::system_clock::from_time_t(time), "", {"core", "feature"}); + CHECK(x.build_xml(t3) == R"( + + + + + + + + + + + + + + + + + + + + + + + + + + + +)"); +} diff --git a/src/vcpkg/ci-baseline.cpp b/src/vcpkg/ci-baseline.cpp index e0bc2fe654..69704d5f74 100644 --- a/src/vcpkg/ci-baseline.cpp +++ b/src/vcpkg/ci-baseline.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -11,11 +12,27 @@ namespace { DECLARE_AND_REGISTER_MESSAGE(ExpectedPortName, (), "", "expected a port name here"); DECLARE_AND_REGISTER_MESSAGE(ExpectedTripletName, (), "", "expected a triplet name here"); - DECLARE_AND_REGISTER_MESSAGE(ExpectedFailOrSkip, (), "", "expected 'fail' or 'skip' here"); + DECLARE_AND_REGISTER_MESSAGE(ExpectedFailOrSkip, (), "", "expected 'fail', 'skip', or 'pass' here"); DECLARE_AND_REGISTER_MESSAGE(UnknownBaselineFileContent, (), "", "unrecognizable baseline entry; expected 'port:triplet=(fail|skip)'"); + + DECLARE_AND_REGISTER_MESSAGE( + CiBaselineRegression, + (msg::spec, msg::build_result, msg::path), + "", + "REGRESSION: {spec} failed with {build_result}. If expected, add {spec}=fail to {path}."); + + DECLARE_AND_REGISTER_MESSAGE(CiBaselineUnexpectedPass, + (msg::spec, msg::path), + "", + "PASSING, REMOVE FROM FAIL LIST: {spec} ({path})."); + + DECLARE_AND_REGISTER_MESSAGE(CiBaselineDisallowedCascade, + (msg::spec, msg::path), + "", + "REGRESSION: {spec} cascaded, but it is required to pass. ({path})."); } namespace vcpkg @@ -116,6 +133,7 @@ namespace vcpkg static constexpr StringLiteral FAIL = "fail"; static constexpr StringLiteral SKIP = "skip"; + static constexpr StringLiteral PASS = "pass"; CiBaselineState state; if (parser.try_match_keyword(FAIL)) { @@ -125,6 +143,10 @@ namespace vcpkg { state = CiBaselineState::Skip; } + else if (parser.try_match_keyword(PASS)) + { + state = CiBaselineState::Pass; + } else { parser.add_error(msg::format(msgExpectedFailOrSkip)); @@ -157,11 +179,12 @@ namespace vcpkg return result; } - SortedVector parse_and_apply_ci_baseline(View lines, - ExclusionsMap& exclusions_map, - SkipFailures skip_failures) + CiBaselineData parse_and_apply_ci_baseline(View lines, + ExclusionsMap& exclusions_map, + SkipFailures skip_failures) { std::vector expected_failures; + std::vector required_success; std::map> added_exclusions; for (const auto& triplet_entry : exclusions_map.triplets) { @@ -174,6 +197,11 @@ namespace vcpkg auto triplet_match = added_exclusions.find(line.triplet); if (triplet_match != added_exclusions.end()) { + if (line.state == CiBaselineState::Pass) + { + required_success.emplace_back(line.port_name, line.triplet); + continue; + } if (line.state == CiBaselineState::Fail) { expected_failures.emplace_back(line.port_name, line.triplet); @@ -193,6 +221,44 @@ namespace vcpkg SortedVector(std::move(added_exclusions.find(triplet_entry.triplet)->second))); } - return SortedVector{std::move(expected_failures)}; + return CiBaselineData{ + SortedVector(std::move(expected_failures)), + SortedVector(std::move(required_success)), + }; + } + + LocalizedString format_ci_result(const PackageSpec& spec, + Build::BuildResult result, + const CiBaselineData& cidata, + StringView cifile, + bool allow_unexpected_passing) + { + switch (result) + { + case Build::BuildResult::BUILD_FAILED: + case Build::BuildResult::POST_BUILD_CHECKS_FAILED: + case Build::BuildResult::FILE_CONFLICTS: + if (!cidata.expected_failures.contains(spec)) + { + return msg::format(msgCiBaselineRegression, + msg::spec = spec, + msg::build_result = Build::to_string_locale_invariant(result), + msg::path = cifile); + } + break; + case Build::BuildResult::SUCCEEDED: + if (!allow_unexpected_passing && cidata.expected_failures.contains(spec)) + { + return msg::format(msgCiBaselineUnexpectedPass, msg::spec = spec, msg::path = cifile); + } + break; + case Build::BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES: + if (cidata.required_success.contains(spec)) + { + return msg::format(msgCiBaselineDisallowedCascade, msg::spec = spec, msg::path = cifile); + } + default: break; + } + return {}; } } diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 04022c7e59..8148d43bca 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include @@ -87,17 +88,6 @@ namespace "Printed before a series of CiBaselineRegression and/or CiBaselineUnexpectedPass messages.", "REGRESSIONS:"); - DECLARE_AND_REGISTER_MESSAGE( - CiBaselineRegression, - (msg::spec, msg::build_result, msg::path), - "", - "REGRESSION: {spec} failed with {build_result}. If expected, add {spec}=fail to {path}."); - - DECLARE_AND_REGISTER_MESSAGE(CiBaselineUnexpectedPass, - (msg::spec, msg::path), - "", - "PASSING, REMOVE FROM FAIL LIST: {spec} ({path})."); - DECLARE_AND_REGISTER_MESSAGE( CiBaselineAllowUnexpectedPassingRequiresBaseline, (), @@ -158,168 +148,6 @@ namespace vcpkg::Commands::CI nullptr, }; - // https://xunit.net/docs/format-xml-v2 - struct XunitTestResults - { - public: - void add_test_results(const PackageSpec& spec, - Build::BuildResult build_result, - const ElapsedTime& elapsed_time, - const std::chrono::system_clock::time_point& start_time, - const std::string& abi_tag, - const std::vector& features) - { - m_tests[spec.name()].push_back( - {spec.to_string(), - Strings::concat(spec.name(), '[', Strings::join(",", features), "]:", spec.triplet()), - spec.triplet().to_string(), - build_result, - elapsed_time, - start_time, - abi_tag, - features}); - } - - std::string build_xml(Triplet controlling_triplet) - { - XmlSerializer xml; - xml.emit_declaration(); - xml.open_tag("assemblies").line_break(); - for (const auto& test_group : m_tests) - { - const auto& port_name = test_group.first; - const auto& port_results = test_group.second; - - ElapsedTime elapsed_sum{}; - for (auto&& port_result : port_results) - { - elapsed_sum += port_result.time; - } - - const auto elapsed_seconds = elapsed_sum.as().count(); - - auto earliest_start_time = std::min_element(port_results.begin(), - port_results.end(), - [](const XunitTest& lhs, const XunitTest& rhs) { - return lhs.start_time < rhs.start_time; - }) - ->start_time; - - const auto as_time_t = std::chrono::system_clock::to_time_t(earliest_start_time); - const auto as_tm = to_utc_time(as_time_t).value_or_exit(VCPKG_LINE_INFO); - char run_date_time[80]; - strftime(run_date_time, sizeof(run_date_time), "%Y-%m-%d%H:%M:%S", &as_tm); - - StringView run_date{run_date_time, 10}; - StringView run_time{run_date_time + 10, 8}; - - xml.start_complex_open_tag("assembly") - .attr("name", port_name) - .attr("run-date", run_date) - .attr("run-time", run_time) - .attr("time", elapsed_seconds) - .finish_complex_open_tag() - .line_break(); - xml.start_complex_open_tag("collection") - .attr("name", controlling_triplet) - .attr("time", elapsed_seconds) - .finish_complex_open_tag() - .line_break(); - for (const auto& port_result : port_results) - { - xml_test(xml, port_result); - } - xml.close_tag("collection").line_break(); - xml.close_tag("assembly").line_break(); - } - - xml.close_tag("assemblies").line_break(); - return std::move(xml.buf); - } - - private: - struct XunitTest - { - std::string name; - std::string method; - std::string owner; - vcpkg::Build::BuildResult result; - vcpkg::ElapsedTime time; - std::chrono::system_clock::time_point start_time; - std::string abi_tag; - std::vector features; - }; - - static void xml_test(XmlSerializer& xml, const XunitTest& test) - { - StringLiteral result_string = ""; - switch (test.result) - { - case BuildResult::POST_BUILD_CHECKS_FAILED: - case BuildResult::FILE_CONFLICTS: - case BuildResult::BUILD_FAILED: result_string = "Fail"; break; - case BuildResult::EXCLUDED: - case BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES: result_string = "Skip"; break; - case BuildResult::SUCCEEDED: result_string = "Pass"; break; - default: Checks::unreachable(VCPKG_LINE_INFO); - } - - xml.start_complex_open_tag("test") - .attr("name", test.name) - .attr("method", test.method) - .attr("time", test.time.as().count()) - .attr("method", result_string) - .finish_complex_open_tag() - .line_break(); - xml.open_tag("traits").line_break(); - if (!test.abi_tag.empty()) - { - xml.start_complex_open_tag("trait") - .attr("name", "abi_tag") - .attr("value", test.abi_tag) - .finish_self_closing_complex_tag() - .line_break(); - } - - if (!test.features.empty()) - { - xml.start_complex_open_tag("trait") - .attr("name", "features") - .attr("value", Strings::join(", ", test.features)) - .finish_self_closing_complex_tag() - .line_break(); - } - - xml.start_complex_open_tag("trait") - .attr("name", "owner") - .attr("value", test.owner) - .finish_self_closing_complex_tag() - .line_break(); - xml.close_tag("traits").line_break(); - - if (result_string == "Fail") - { - xml.open_tag("failure") - .open_tag("message") - .cdata(to_string_locale_invariant(test.result)) - .close_tag("failure") - .close_tag("message") - .line_break(); - } - else if (result_string == "Skip") - { - xml.open_tag("reason").cdata(to_string_locale_invariant(test.result)).close_tag("reason").line_break(); - } - else - { - Checks::check_exit(VCPKG_LINE_INFO, result_string == "Pass"); - } - xml.close_tag("test").line_break(); - } - - std::map> m_tests; - }; - struct UnknownCIPortsResults { std::map known; @@ -490,7 +318,7 @@ namespace vcpkg::Commands::CI } static void print_baseline_regressions(const TripletAndSummary& result, - const SortedVector& expected_failures, + const CiBaselineData& cidata, const std::string& ci_baseline_file_name, bool allow_unexpected_passing) { @@ -498,32 +326,8 @@ namespace vcpkg::Commands::CI for (auto&& port_result : result.summary.results) { auto& build_result = port_result.build_result.value_or_exit(VCPKG_LINE_INFO); - switch (build_result.code) - { - case Build::BuildResult::BUILD_FAILED: - case Build::BuildResult::POST_BUILD_CHECKS_FAILED: - case Build::BuildResult::FILE_CONFLICTS: - if (!expected_failures.contains(port_result.get_spec())) - { - output.append(msg::format(msgCiBaselineRegression, - msg::spec = port_result.get_spec().to_string(), - msg::build_result = - Build::to_string_locale_invariant(build_result.code).to_string(), - msg::path = ci_baseline_file_name)); - output.append_raw('\n'); - } - break; - case Build::BuildResult::SUCCEEDED: - if (!allow_unexpected_passing && expected_failures.contains(port_result.get_spec())) - { - output.append(msg::format(msgCiBaselineUnexpectedPass, - msg::spec = port_result.get_spec().to_string(), - msg::path = ci_baseline_file_name)); - output.append_raw('\n'); - } - break; - default: break; - } + output.append(format_ci_result( + port_result.get_spec(), build_result.code, cidata, ci_baseline_file_name, allow_unexpected_passing)); } auto output_data = output.extract_data(); @@ -556,7 +360,7 @@ namespace vcpkg::Commands::CI parse_exclusions(settings, OPTION_HOST_EXCLUDE, host_triplet, exclusions_map); auto baseline_iter = settings.find(OPTION_CI_BASELINE); const bool allow_unexpected_passing = Util::Sets::contains(options.switches, OPTION_ALLOW_UNEXPECTED_PASSING); - SortedVector expected_failures; + CiBaselineData cidata; if (baseline_iter == settings.end()) { if (allow_unexpected_passing) @@ -574,7 +378,7 @@ namespace vcpkg::Commands::CI ParseMessages ci_parse_messages; const auto lines = parse_ci_baseline(ci_baseline_file_contents, ci_baseline_file_name, ci_parse_messages); ci_parse_messages.exit_if_errors_or_warnings(ci_baseline_file_name); - expected_failures = parse_and_apply_ci_baseline(lines, exclusions_map, skip_failures); + cidata = parse_and_apply_ci_baseline(lines, exclusions_map, skip_failures); } auto skipped_cascade_count = parse_skipped_cascade_count(settings); @@ -603,7 +407,7 @@ namespace vcpkg::Commands::CI std::vector> all_known_results; - XunitTestResults xunitTestResults; + XunitWriter xunitTestResults; std::vector results; auto timer = ElapsedTimer::create_started(); @@ -758,7 +562,7 @@ namespace vcpkg::Commands::CI if (baseline_iter != settings.end()) { - print_baseline_regressions(result, expected_failures, baseline_iter->second, allow_unexpected_passing); + print_baseline_regressions(result, cidata, baseline_iter->second, allow_unexpected_passing); } } diff --git a/src/vcpkg/install.cpp b/src/vcpkg/install.cpp index 8ee779d124..96b9ea4e33 100644 --- a/src/vcpkg/install.cpp +++ b/src/vcpkg/install.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include @@ -1254,12 +1255,19 @@ namespace vcpkg::Install auto it_xunit = options.settings.find(OPTION_XUNIT); if (it_xunit != options.settings.end()) { - std::string xunit_doc = "\n"; + XunitWriter xwriter; - xunit_doc += summary.xunit_results(); + for (auto&& result : summary.results) + { + xwriter.add_test_results(result.get_spec(), + result.build_result.value_or_exit(VCPKG_LINE_INFO).code, + result.timing, + result.start_time, + "", + {}); + } - xunit_doc += "\n"; - fs.write_contents(it_xunit->second, xunit_doc, VCPKG_LINE_INFO); + fs.write_contents(it_xunit->second, xwriter.build_xml(default_triplet), VCPKG_LINE_INFO); } std::set printed_usages; @@ -1330,50 +1338,6 @@ namespace vcpkg::Install return m_install_action && m_install_action->request_type == RequestType::USER_REQUESTED; } - static std::string xunit_result(const PackageSpec& spec, ElapsedTime time, BuildResult code) - { - std::string message_block; - const char* result_string = ""; - switch (code) - { - case BuildResult::POST_BUILD_CHECKS_FAILED: - case BuildResult::FILE_CONFLICTS: - case BuildResult::BUILD_FAILED: - case BuildResult::CACHE_MISSING: - result_string = "Fail"; - message_block = Strings::format("", - to_string_locale_invariant(code)); - break; - case BuildResult::EXCLUDED: - case BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES: - result_string = "Skip"; - message_block = Strings::format("", to_string_locale_invariant(code)); - break; - case BuildResult::SUCCEEDED: result_string = "Pass"; break; - default: Checks::unreachable(VCPKG_LINE_INFO); - } - - return Strings::format(R"(%s)" - "\n", - spec, - spec, - time.as().count(), - result_string, - message_block); - } - - std::string InstallSummary::xunit_results() const - { - std::string xunit_doc; - for (auto&& result : results) - { - xunit_doc += - xunit_result(result.get_spec(), result.timing, result.build_result.value_or_exit(VCPKG_LINE_INFO).code); - } - - return xunit_doc; - } - void track_install_plan(Dependencies::ActionPlan& plan) { Cache triplet_hashes; diff --git a/src/vcpkg/xunitwriter.cpp b/src/vcpkg/xunitwriter.cpp new file mode 100644 index 0000000000..9552eec8fa --- /dev/null +++ b/src/vcpkg/xunitwriter.cpp @@ -0,0 +1,168 @@ +#include +#include + +#include +#include + +using namespace vcpkg; + +struct vcpkg::XunitTest +{ + std::string name; + std::string method; + std::string owner; + vcpkg::Build::BuildResult result; + vcpkg::ElapsedTime time; + std::chrono::system_clock::time_point start_time; + std::string abi_tag; + std::vector features; +}; + +namespace +{ + using vcpkg::Build::BuildResult; + + static void xml_test(XmlSerializer& xml, const XunitTest& test) + { + StringLiteral result_string = ""; + switch (test.result) + { + case BuildResult::POST_BUILD_CHECKS_FAILED: + case BuildResult::FILE_CONFLICTS: + case BuildResult::BUILD_FAILED: result_string = "Fail"; break; + case BuildResult::EXCLUDED: + case BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES: result_string = "Skip"; break; + case BuildResult::SUCCEEDED: result_string = "Pass"; break; + default: Checks::unreachable(VCPKG_LINE_INFO); + } + + xml.start_complex_open_tag("test") + .attr("name", test.name) + .attr("method", test.method) + .attr("time", test.time.as().count()) + .attr("method", result_string) + .finish_complex_open_tag() + .line_break(); + xml.open_tag("traits").line_break(); + if (!test.abi_tag.empty()) + { + xml.start_complex_open_tag("trait") + .attr("name", "abi_tag") + .attr("value", test.abi_tag) + .finish_self_closing_complex_tag() + .line_break(); + } + + if (!test.features.empty()) + { + xml.start_complex_open_tag("trait") + .attr("name", "features") + .attr("value", Strings::join(", ", test.features)) + .finish_self_closing_complex_tag() + .line_break(); + } + + xml.start_complex_open_tag("trait") + .attr("name", "owner") + .attr("value", test.owner) + .finish_self_closing_complex_tag() + .line_break(); + xml.close_tag("traits").line_break(); + + if (result_string == "Fail") + { + xml.open_tag("failure") + .open_tag("message") + .cdata(to_string_locale_invariant(test.result)) + .close_tag("failure") + .close_tag("message") + .line_break(); + } + else if (result_string == "Skip") + { + xml.open_tag("reason").cdata(to_string_locale_invariant(test.result)).close_tag("reason").line_break(); + } + else + { + Checks::check_exit(VCPKG_LINE_INFO, result_string == "Pass"); + } + xml.close_tag("test").line_break(); + } + +} + +XunitWriter::XunitWriter() { } +XunitWriter::~XunitWriter() { } + +void XunitWriter::add_test_results(const PackageSpec& spec, + Build::BuildResult build_result, + const ElapsedTime& elapsed_time, + const std::chrono::system_clock::time_point& start_time, + const std::string& abi_tag, + const std::vector& features) +{ + m_tests[spec.name()].push_back( + {spec.to_string(), + Strings::concat(spec.name(), '[', Strings::join(",", features), "]:", spec.triplet()), + spec.triplet().to_string(), + build_result, + elapsed_time, + start_time, + abi_tag, + features}); +} + +std::string XunitWriter::build_xml(Triplet controlling_triplet) +{ + XmlSerializer xml; + xml.emit_declaration(); + xml.open_tag("assemblies").line_break(); + for (const auto& test_group : m_tests) + { + const auto& port_name = test_group.first; + const auto& port_results = test_group.second; + + ElapsedTime elapsed_sum{}; + for (auto&& port_result : port_results) + { + elapsed_sum += port_result.time; + } + + const auto elapsed_seconds = elapsed_sum.as().count(); + + auto earliest_start_time = + std::min_element(port_results.begin(), port_results.end(), [](const XunitTest& lhs, const XunitTest& rhs) { + return lhs.start_time < rhs.start_time; + })->start_time; + + const auto as_time_t = std::chrono::system_clock::to_time_t(earliest_start_time); + const auto as_tm = to_utc_time(as_time_t).value_or_exit(VCPKG_LINE_INFO); + char run_date_time[80]; + strftime(run_date_time, sizeof(run_date_time), "%Y-%m-%d%H:%M:%S", &as_tm); + + StringView run_date{run_date_time, 10}; + StringView run_time{run_date_time + 10, 8}; + + xml.start_complex_open_tag("assembly") + .attr("name", port_name) + .attr("run-date", run_date) + .attr("run-time", run_time) + .attr("time", elapsed_seconds) + .finish_complex_open_tag() + .line_break(); + xml.start_complex_open_tag("collection") + .attr("name", controlling_triplet) + .attr("time", elapsed_seconds) + .finish_complex_open_tag() + .line_break(); + for (const auto& port_result : port_results) + { + xml_test(xml, port_result); + } + xml.close_tag("collection").line_break(); + xml.close_tag("assembly").line_break(); + } + + xml.close_tag("assemblies").line_break(); + return std::move(xml.buf); +} From e0e38ca604a98cc531884c89d1876ee0faba2d9a Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal Date: Thu, 9 Jun 2022 22:42:10 -0700 Subject: [PATCH 2/6] generate-message-map --- locales/messages.en.json | 3 ++- locales/messages.json | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/locales/messages.en.json b/locales/messages.en.json index 5bfec55f71..a480abb7a8 100644 --- a/locales/messages.en.json +++ b/locales/messages.en.json @@ -64,6 +64,7 @@ "ChecksUnreachableCode": "unreachable code was reached", "ChecksUpdateVcpkg": "updating vcpkg by rerunning bootstrap-vcpkg may resolve this failure.", "CiBaselineAllowUnexpectedPassingRequiresBaseline": "--allow-unexpected-passing can only be used if a baseline is provided via --ci-baseline.", + "CiBaselineDisallowedCascade": "REGRESSION: {spec} cascaded, but it is required to pass. ({path}).", "CiBaselineRegression": "REGRESSION: {spec} failed with {build_result}. If expected, add {spec}=fail to {path}.", "CiBaselineRegressionHeader": "REGRESSIONS:", "CiBaselineUnexpectedPass": "PASSING, REMOVE FROM FAIL LIST: {spec} ({path}).", @@ -99,7 +100,7 @@ "ErrorVsCodeNotFoundPathExamined": "The following paths were examined:", "ExcludedPackage": "Excluded {spec}", "ExpectedCharacterHere": "expected '{expected}' here", - "ExpectedFailOrSkip": "expected 'fail' or 'skip' here", + "ExpectedFailOrSkip": "expected 'fail', 'skip', or 'pass' here", "ExpectedPortName": "expected a port name here", "ExpectedTripletName": "expected a triplet name here", "FailedToProvisionCe": "Failed to provision vcpkg-ce.", diff --git a/locales/messages.json b/locales/messages.json index 5889097543..ce83437267 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -117,6 +117,8 @@ "ChecksUnreachableCode": "unreachable code was reached", "ChecksUpdateVcpkg": "updating vcpkg by rerunning bootstrap-vcpkg may resolve this failure.", "CiBaselineAllowUnexpectedPassingRequiresBaseline": "--allow-unexpected-passing can only be used if a baseline is provided via --ci-baseline.", + "CiBaselineDisallowedCascade": "REGRESSION: {spec} cascaded, but it is required to pass. ({path}).", + "_CiBaselineDisallowedCascade.comment": "An example of {spec} is zlib:x64-windows. An example of {path} is /foo/bar.", "CiBaselineRegression": "REGRESSION: {spec} failed with {build_result}. If expected, add {spec}=fail to {path}.", "_CiBaselineRegression.comment": "An example of {spec} is zlib:x64-windows. An example of {build_result} is One of the BuildResultXxx messages (such as BuildResultSucceeded/SUCCEEDED). An example of {path} is /foo/bar.", "CiBaselineRegressionHeader": "REGRESSIONS:", @@ -179,7 +181,7 @@ "_ExcludedPackage.comment": "An example of {spec} is zlib:x64-windows.", "ExpectedCharacterHere": "expected '{expected}' here", "_ExpectedCharacterHere.comment": "{expected} is a locale-invariant delimiter; for example, the ':' or '=' in 'zlib:x64-windows=skip'", - "ExpectedFailOrSkip": "expected 'fail' or 'skip' here", + "ExpectedFailOrSkip": "expected 'fail', 'skip', or 'pass' here", "ExpectedPortName": "expected a port name here", "ExpectedTripletName": "expected a triplet name here", "FailedToProvisionCe": "Failed to provision vcpkg-ce.", From a55e6ece90a63faf9803789bc27b89c3554c1f06 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Mon, 13 Jun 2022 23:50:06 +0000 Subject: [PATCH 3/6] PR comments --- include/vcpkg-test/util.h | 8 ++++---- include/vcpkg/fwd/triplet.h | 2 +- src/vcpkg/ci-baseline.cpp | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/vcpkg-test/util.h b/include/vcpkg-test/util.h index 4e1e50f72a..a240a6eecc 100644 --- a/include/vcpkg-test/util.h +++ b/include/vcpkg-test/util.h @@ -111,14 +111,14 @@ namespace vcpkg::Test const char* depends = "", const char* triplet = "x86-windows"); - extern const Triplet ARM64_WINDOWS; + extern const Triplet X86_WINDOWS; extern const Triplet X64_WINDOWS; extern const Triplet X64_WINDOWS_STATIC; extern const Triplet X64_WINDOWS_STATIC_MD; - extern const Triplet X86_WINDOWS; - extern const Triplet ARM_UWP; - extern const Triplet X64_UWP; + extern const Triplet ARM64_WINDOWS; extern const Triplet X86_UWP; + extern const Triplet X64_UWP; + extern const Triplet ARM_UWP; extern const Triplet X64_ANDROID; extern const Triplet X64_OSX; extern const Triplet X64_LINUX; diff --git a/include/vcpkg/fwd/triplet.h b/include/vcpkg/fwd/triplet.h index 3e9e1bacfa..413e6391eb 100644 --- a/include/vcpkg/fwd/triplet.h +++ b/include/vcpkg/fwd/triplet.h @@ -3,4 +3,4 @@ namespace vcpkg { struct Triplet; -} \ No newline at end of file +} diff --git a/src/vcpkg/ci-baseline.cpp b/src/vcpkg/ci-baseline.cpp index 69704d5f74..ac4d345e15 100644 --- a/src/vcpkg/ci-baseline.cpp +++ b/src/vcpkg/ci-baseline.cpp @@ -16,7 +16,7 @@ namespace DECLARE_AND_REGISTER_MESSAGE(UnknownBaselineFileContent, (), "", - "unrecognizable baseline entry; expected 'port:triplet=(fail|skip)'"); + "unrecognizable baseline entry; expected 'port:triplet=(fail|skip|pass)'"); DECLARE_AND_REGISTER_MESSAGE( CiBaselineRegression, From 79e6a4483e58fa684ea279a008b7d66b4ba5cf9a Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Tue, 14 Jun 2022 01:01:29 +0000 Subject: [PATCH 4/6] Fix tests --- src/vcpkg-test/ci-baseline.cpp | 6 +++--- src/vcpkg-test/xunitwriter.cpp | 12 ++++++------ src/vcpkg/commands.ci.cpp | 22 +++++++++++++--------- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/vcpkg-test/ci-baseline.cpp b/src/vcpkg-test/ci-baseline.cpp index 596bf11ace..ab2880d0a3 100644 --- a/src/vcpkg-test/ci-baseline.cpp +++ b/src/vcpkg-test/ci-baseline.cpp @@ -305,18 +305,18 @@ TEST_CASE ("Parse Errors", "[ci-baseline]") ^)"); check_error("example:x64-windows = fail extra stuff", - R"(test:1:33: error: unrecognizable baseline entry; expected 'port:triplet=(fail|skip)' + R"(test:1:33: error: unrecognizable baseline entry; expected 'port:triplet=(fail|skip|pass)' on expression: example:x64-windows = fail extra stuff ^)"); check_error("example:x64-windows = fail example:x64-windows = fail", - R"(test:1:33: error: unrecognizable baseline entry; expected 'port:triplet=(fail|skip)' + R"(test:1:33: error: unrecognizable baseline entry; expected 'port:triplet=(fail|skip|pass)' on expression: example:x64-windows = fail example:x64-windows = fail ^)"); check_error("example:x64-windows = fail # extra stuff\n" "example:x64-uwp=skip extra stuff\n", - R"(test:2:22: error: unrecognizable baseline entry; expected 'port:triplet=(fail|skip)' + R"(test:2:22: error: unrecognizable baseline entry; expected 'port:triplet=(fail|skip|pass)' on expression: example:x64-uwp=skip extra stuff ^)"); } diff --git a/src/vcpkg-test/xunitwriter.cpp b/src/vcpkg-test/xunitwriter.cpp index 46abe41aa3..ceab190879 100644 --- a/src/vcpkg-test/xunitwriter.cpp +++ b/src/vcpkg-test/xunitwriter.cpp @@ -19,11 +19,11 @@ TEST_CASE ("Simple XunitWriter", "[xunitwriter]") CHECK(x.build_xml(t) == R"( - + - + @@ -49,23 +49,23 @@ TEST_CASE ("XunitWriter Two", "[xunitwriter]") CHECK(x.build_xml(t3) == R"( - + - + - + - + diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 2f6cdd908d..9511c57f85 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -324,23 +324,27 @@ namespace vcpkg::Commands::CI const std::string& ci_baseline_file_name, bool allow_unexpected_passing) { - LocalizedString output; + bool has_error = false; + LocalizedString output = msg::format(msgCiBaselineRegressionHeader); + output.append_raw('\n'); for (auto&& port_result : result.summary.results) { auto& build_result = port_result.build_result.value_or_exit(VCPKG_LINE_INFO); - output.append(format_ci_result( - port_result.get_spec(), build_result.code, cidata, ci_baseline_file_name, allow_unexpected_passing)); + + auto msg = format_ci_result( + port_result.get_spec(), build_result.code, cidata, ci_baseline_file_name, allow_unexpected_passing); + if (!msg.empty()) + { + has_error = true; + output.append(msg).append_raw("\n"); + } } - auto output_data = output.extract_data(); - if (output_data.empty()) + if (!has_error) { return; } - - LocalizedString header = msg::format(msgCiBaselineRegressionHeader); - header.append_raw('\n'); - output_data.insert(0, header.extract_data()); + auto output_data = output.extract_data(); fwrite(output_data.data(), 1, output_data.size(), stderr); } From e3ca48acacff5fd95857e36a5c5704b26010475d Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Tue, 14 Jun 2022 01:11:04 +0000 Subject: [PATCH 5/6] Update message map --- locales/messages.en.json | 2 +- locales/messages.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/locales/messages.en.json b/locales/messages.en.json index a480abb7a8..a4b0ef92bf 100644 --- a/locales/messages.en.json +++ b/locales/messages.en.json @@ -195,7 +195,7 @@ "ToolInWin10": "This utility is bundled with Windows 10 or later.", "UnexpectedErrorDuringBulkDownload": "an unexpected error occurred during bulk download.", "UnexpectedToolOutput": "{tool_name} ({path}) produced unexpected output when attempting to determine the version:", - "UnknownBaselineFileContent": "unrecognizable baseline entry; expected 'port:triplet=(fail|skip)'", + "UnknownBaselineFileContent": "unrecognizable baseline entry; expected 'port:triplet=(fail|skip|pass)'", "UnknownBinaryProviderType": "unknown binary provider type: valid providers are 'clear', 'default', 'nuget', 'nugetconfig', 'interactive', and 'files'", "UnknownTool": "vcpkg does not have a definition of this tool for this platform.", "UnsupportedSystemName": "Could not map VCPKG_CMAKE_SYSTEM_NAME '{system_name}' to a vcvarsall platform. Supported system names are '', 'Windows' and 'WindowsStore'.", diff --git a/locales/messages.json b/locales/messages.json index ce83437267..247b32a059 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -338,7 +338,7 @@ "UnexpectedErrorDuringBulkDownload": "an unexpected error occurred during bulk download.", "UnexpectedToolOutput": "{tool_name} ({path}) produced unexpected output when attempting to determine the version:", "_UnexpectedToolOutput.comment": "The actual command line output will be appended after this message. An example of {tool_name} is aria2. An example of {path} is /foo/bar.", - "UnknownBaselineFileContent": "unrecognizable baseline entry; expected 'port:triplet=(fail|skip)'", + "UnknownBaselineFileContent": "unrecognizable baseline entry; expected 'port:triplet=(fail|skip|pass)'", "UnknownBinaryProviderType": "unknown binary provider type: valid providers are 'clear', 'default', 'nuget', 'nugetconfig', 'interactive', and 'files'", "UnknownTool": "vcpkg does not have a definition of this tool for this platform.", "UnsupportedSystemName": "Could not map VCPKG_CMAKE_SYSTEM_NAME '{system_name}' to a vcvarsall platform. Supported system names are '', 'Windows' and 'WindowsStore'.", From 004b64f694a98cfde32ac7f24d231abee44460e1 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Tue, 14 Jun 2022 02:04:57 +0000 Subject: [PATCH 6/6] Report cascade failures on known results --- src/vcpkg/commands.ci.cpp | 50 ++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 9511c57f85..e8442d2d68 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -319,7 +319,7 @@ namespace vcpkg::Commands::CI return result; } - static void print_baseline_regressions(const TripletAndSummary& result, + static void print_baseline_regressions(const std::map& results, const CiBaselineData& cidata, const std::string& ci_baseline_file_name, bool allow_unexpected_passing) @@ -327,12 +327,9 @@ namespace vcpkg::Commands::CI bool has_error = false; LocalizedString output = msg::format(msgCiBaselineRegressionHeader); output.append_raw('\n'); - for (auto&& port_result : result.summary.results) + for (auto&& r : results) { - auto& build_result = port_result.build_result.value_or_exit(VCPKG_LINE_INFO); - - auto msg = format_ci_result( - port_result.get_spec(), build_result.code, cidata, ci_baseline_file_name, allow_unexpected_passing); + auto msg = format_ci_result(r.first, r.second, cidata, ci_baseline_file_name, allow_unexpected_passing); if (!msg.empty()) { has_error = true; @@ -411,11 +408,8 @@ namespace vcpkg::Commands::CI auto var_provider_storage = CMakeVars::make_triplet_cmake_var_provider(paths); auto& var_provider = *var_provider_storage; - std::vector> all_known_results; - XunitWriter xunitTestResults; - std::vector results; auto timer = ElapsedTimer::create_started(); std::vector all_port_names = Util::fmap(provider.load_all_control_files(), Paragraphs::get_name_of_control_file); @@ -530,18 +524,20 @@ namespace vcpkg::Commands::CI build_logs_recorder, var_provider); + std::map full_results; + // Adding results for ports that were built or pulled from an archive for (auto&& result : summary.results) { - auto& port_features = split_specs->features.at(result.get_spec()); - split_specs->known.erase(result.get_spec()); - xunitTestResults.add_test_results(result.get_spec(), - result.build_result.value_or_exit(VCPKG_LINE_INFO).code, - result.timing, - result.start_time, - split_specs->abi_map.at(result.get_spec()), - port_features); + const auto& spec = result.get_spec(); + auto& port_features = split_specs->features.at(spec); + split_specs->known.erase(spec); + auto code = result.build_result.value_or_exit(VCPKG_LINE_INFO).code; + xunitTestResults.add_test_results( + spec, code, result.timing, result.start_time, split_specs->abi_map.at(spec), port_features); + full_results.emplace(spec, code); } + full_results.insert(split_specs->known.begin(), split_specs->known.end()); // Adding results for ports that were not built because they have known states if (Util::Sets::contains(options.switches, OPTION_XUNIT_ALL)) @@ -558,27 +554,23 @@ namespace vcpkg::Commands::CI } } - all_known_results.emplace_back(std::move(split_specs->known)); - - results.push_back({target_triplet, std::move(summary)}); - } + TripletAndSummary result{target_triplet, std::move(summary)}; - for (auto&& result : results) - { print2("\nTriplet: ", result.triplet, "\n"); print2("Total elapsed time: ", GlobalState::timer.to_string(), "\n"); result.summary.print(); if (baseline_iter != settings.end()) { - print_baseline_regressions(result, cidata, baseline_iter->second, allow_unexpected_passing); + print_baseline_regressions(full_results, cidata, baseline_iter->second, allow_unexpected_passing); } - } - auto it_xunit = settings.find(OPTION_XUNIT); - if (it_xunit != settings.end()) - { - filesystem.write_contents(it_xunit->second, xunitTestResults.build_xml(target_triplet), VCPKG_LINE_INFO); + auto it_xunit = settings.find(OPTION_XUNIT); + if (it_xunit != settings.end()) + { + filesystem.write_contents( + it_xunit->second, xunitTestResults.build_xml(target_triplet), VCPKG_LINE_INFO); + } } Checks::exit_success(VCPKG_LINE_INFO);