From c17e406a1c56de0fbd7028590727c297d903a21c Mon Sep 17 00:00:00 2001 From: Leander Schulten Date: Mon, 7 Feb 2022 20:11:22 +0100 Subject: [PATCH 01/23] vcpkg ci: Parse ci.baseline.txt to determine regressions --- src/vcpkg/commands.ci.cpp | 98 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 94 insertions(+), 4 deletions(-) diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 5efb217525..89211397df 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -25,6 +26,8 @@ #include #include +#include + using namespace vcpkg; namespace @@ -74,6 +77,41 @@ namespace private: Path base_path; }; + + enum class CiBaselineValue + { + Skip, + Fail + }; + + template + void parse_ci_baseline(View lines, F&& callback) + { + for (auto&& line : lines) + { + if (line.empty() || line[0] == '#') continue; + auto colon_sign = line.find(':'); + Checks::check_exit(VCPKG_LINE_INFO, colon_sign != std::string::npos, "Line '%s' must contain a ':'", line); + auto equal_sign = line.find('=', colon_sign + 1); + Checks::check_exit(VCPKG_LINE_INFO, equal_sign != std::string::npos, "Line '%s' must contain a '='", line); + auto port = Strings::trim(StringView(line.c_str(), colon_sign)); + auto triplet = Strings::trim(StringView(line.c_str() + colon_sign + 1, equal_sign - colon_sign - 1)); + auto baseline_value = + Strings::trim(StringView(line.c_str() + equal_sign + 1, line.size() - equal_sign - 1)); + if (baseline_value == "fail") + { + callback(port, triplet, CiBaselineValue::Fail); + } + else if (baseline_value == "skip") + { + callback(port, triplet, CiBaselineValue::Skip); + } + else + { + Checks::exit_with_message(VCPKG_LINE_INFO, "Unknown value '%s'", baseline_value); + } + } + } } namespace vcpkg::Commands::CI @@ -93,15 +131,17 @@ namespace vcpkg::Commands::CI static constexpr StringLiteral OPTION_HOST_EXCLUDE = "host-exclude"; static constexpr StringLiteral OPTION_FAILURE_LOGS = "failure-logs"; static constexpr StringLiteral OPTION_XUNIT = "x-xunit"; + static constexpr StringLiteral OPTION_CI_BASELINE = "ci-baseline"; static constexpr StringLiteral OPTION_RANDOMIZE = "x-randomize"; static constexpr StringLiteral OPTION_OUTPUT_HASHES = "output-hashes"; static constexpr StringLiteral OPTION_PARENT_HASHES = "parent-hashes"; static constexpr StringLiteral OPTION_SKIPPED_CASCADE_COUNT = "x-skipped-cascade-count"; - static constexpr std::array CI_SETTINGS = { + static constexpr std::array CI_SETTINGS = { {{OPTION_EXCLUDE, "Comma separated list of ports to skip"}, {OPTION_HOST_EXCLUDE, "Comma separated list of ports to skip for the host triplet"}, {OPTION_XUNIT, "File to output results in XUnit format (internal)"}, + {OPTION_CI_BASELINE, "Path to the ci.baseline.txt file. Used to skip ports and detect regressions."}, {OPTION_FAILURE_LOGS, "Directory to which failure logs will be copied"}, {OPTION_OUTPUT_HASHES, "File to output all determined package hashes"}, {OPTION_PARENT_HASHES, @@ -483,10 +523,38 @@ namespace vcpkg::Commands::CI const auto& settings = options.settings; BinaryCache binary_cache{args, paths}; - Triplet target_triplet = Triplet::from_canonical_name(std::string(args.command_arguments[0])); + Triplet target_triplet = Triplet::from_canonical_name(args.command_arguments[0]); + + auto exclusions = parse_exclusions(settings, OPTION_EXCLUDE); + auto host_exclusions = parse_exclusions(settings, OPTION_HOST_EXCLUDE); + auto baseline_iter = settings.find(OPTION_CI_BASELINE); + std::set expected_failures; + if (baseline_iter != settings.end()) + { + parse_ci_baseline(paths.get_filesystem().read_lines(baseline_iter->second, VCPKG_LINE_INFO), + [&, + target_triplet_string = args.command_arguments[0], + host_triplet_string = host_triplet.canonical_name()]( + StringView port, StringView triplet, CiBaselineValue value) { + if (triplet == target_triplet_string) + { + if (value == CiBaselineValue::Skip) + expected_failures.emplace(port.to_string(), target_triplet); + else + exclusions.insert(port.to_string()); + } + else if (triplet == host_triplet_string) + { + if (value == CiBaselineValue::Skip) + expected_failures.emplace(port.to_string(), host_triplet); + else + host_exclusions.insert(port.to_string()); + } + }); + } ExclusionPredicate is_excluded{ - parse_exclusions(settings, OPTION_EXCLUDE), - parse_exclusions(settings, OPTION_HOST_EXCLUDE), + exclusions, + host_exclusions, target_triplet, host_triplet, }; @@ -677,6 +745,28 @@ namespace vcpkg::Commands::CI print2("\nTriplet: ", result.triplet, "\n"); print2("Total elapsed time: ", GlobalState::timer.to_string(), "\n"); result.summary.print(); + + if (baseline_iter != settings.end()) + { + print2("\nREGRESSIONS:\n"); + for (auto&& port_result : result.summary.results) + { + switch (port_result.build_result.code) + { + case Build::BuildResult::BUILD_FAILED: + case Build::BuildResult::POST_BUILD_CHECKS_FAILED: + case Build::BuildResult::FILE_CONFLICTS: + if (expected_failures.find(port_result.spec) == expected_failures.end()) + { + std::cerr << " REGRESSION: " << port_result.spec.to_string() << " failed with " + << Build::to_string(port_result.build_result.code) << ". If expected, add " + << port_result.spec.to_string() << "=fail to " << baseline_iter->second + << std::endl; + } + default: break; + } + } + } } auto it_xunit = settings.find(OPTION_XUNIT); From a5556453e8be7aea1e6bddce7c940cb2673bac89 Mon Sep 17 00:00:00 2001 From: Leander Schulten Date: Fri, 11 Feb 2022 04:40:40 +0100 Subject: [PATCH 02/23] Fix bug and implement PASSING, REMOVE FROM FAIL LIST and --passing-is-passing --- src/vcpkg/commands.ci.cpp | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 89211397df..ca7a986c4e 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -132,6 +132,7 @@ namespace vcpkg::Commands::CI static constexpr StringLiteral OPTION_FAILURE_LOGS = "failure-logs"; static constexpr StringLiteral OPTION_XUNIT = "x-xunit"; static constexpr StringLiteral OPTION_CI_BASELINE = "ci-baseline"; + static constexpr StringLiteral OPTION_PASSING_IS_PASSING = "passing-is-passing"; static constexpr StringLiteral OPTION_RANDOMIZE = "x-randomize"; static constexpr StringLiteral OPTION_OUTPUT_HASHES = "output-hashes"; static constexpr StringLiteral OPTION_PARENT_HASHES = "parent-hashes"; @@ -149,9 +150,10 @@ namespace vcpkg::Commands::CI {OPTION_SKIPPED_CASCADE_COUNT, "Asserts that the number of --exclude and supports skips exactly equal this number"}}}; - static constexpr std::array CI_SWITCHES = {{ + static constexpr std::array CI_SWITCHES = {{ {OPTION_DRY_RUN, "Print out plan without execution"}, {OPTION_RANDOMIZE, "Randomize the install order"}, + {OPTION_PASSING_IS_PASSING, "Indicates that 'Passing, remove from fail list' results should not be emitted."}, }}; const CommandStructure COMMAND_STRUCTURE = { @@ -528,6 +530,7 @@ namespace vcpkg::Commands::CI auto exclusions = parse_exclusions(settings, OPTION_EXCLUDE); auto host_exclusions = parse_exclusions(settings, OPTION_HOST_EXCLUDE); auto baseline_iter = settings.find(OPTION_CI_BASELINE); + const bool passing_is_passing = Util::Sets::contains(options.switches, OPTION_PASSING_IS_PASSING); std::set expected_failures; if (baseline_iter != settings.end()) { @@ -539,19 +542,28 @@ namespace vcpkg::Commands::CI if (triplet == target_triplet_string) { if (value == CiBaselineValue::Skip) - expected_failures.emplace(port.to_string(), target_triplet); - else exclusions.insert(port.to_string()); + else if (value == CiBaselineValue::Fail) + expected_failures.emplace(port.to_string(), target_triplet); } else if (triplet == host_triplet_string) { if (value == CiBaselineValue::Skip) - expected_failures.emplace(port.to_string(), host_triplet); - else host_exclusions.insert(port.to_string()); + else if (value == CiBaselineValue::Fail) + expected_failures.emplace(port.to_string(), host_triplet); } }); } + else + { + Checks::check_exit(VCPKG_LINE_INFO, + !passing_is_passing, + Strings::concat("--", + OPTION_PASSING_IS_PASSING, + " can only be used if a ci baseline is provided via --", + OPTION_CI_BASELINE)); + } ExclusionPredicate is_excluded{ exclusions, host_exclusions, @@ -763,6 +775,15 @@ namespace vcpkg::Commands::CI << port_result.spec.to_string() << "=fail to " << baseline_iter->second << std::endl; } + break; + case Build::BuildResult::SUCCEEDED: + if (!passing_is_passing && + expected_failures.find(port_result.spec) != expected_failures.end()) + { + std::cerr << " PASSING, REMOVE FROM FAIL LIST: " << port_result.spec.to_string() + << " (" << baseline_iter->second << ")" << std::endl; + } + break; default: break; } } From fc2cbb5c576b54cbd89435440cfa41cddc3a777d Mon Sep 17 00:00:00 2001 From: Leander Schulten Date: Sun, 6 Mar 2022 13:39:56 +0100 Subject: [PATCH 03/23] Apply Nicole's CR --- src/vcpkg/commands.ci.cpp | 103 +++++++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 41 deletions(-) diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index ca7a986c4e..7a9385d284 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -78,39 +78,51 @@ namespace Path base_path; }; - enum class CiBaselineValue + enum class CiBaselineState { Skip, - Fail + Fail, }; - template - void parse_ci_baseline(View lines, F&& callback) + struct CiBaselineLine { - for (auto&& line : lines) + std::string port_name; + std::string triplet_name; + CiBaselineState state; + }; + + std::vector parse_ci_baseline(View lines) + { + std::vector result; + for (auto& line : lines) { if (line.empty() || line[0] == '#') continue; - auto colon_sign = line.find(':'); - Checks::check_exit(VCPKG_LINE_INFO, colon_sign != std::string::npos, "Line '%s' must contain a ':'", line); - auto equal_sign = line.find('=', colon_sign + 1); - Checks::check_exit(VCPKG_LINE_INFO, equal_sign != std::string::npos, "Line '%s' must contain a '='", line); - auto port = Strings::trim(StringView(line.c_str(), colon_sign)); - auto triplet = Strings::trim(StringView(line.c_str() + colon_sign + 1, equal_sign - colon_sign - 1)); - auto baseline_value = - Strings::trim(StringView(line.c_str() + equal_sign + 1, line.size() - equal_sign - 1)); + CiBaselineLine parsed_line; + + auto colon_loc = line.find(':'); + Checks::check_exit(VCPKG_LINE_INFO, colon_loc != std::string::npos, "Line '%s' must contain a ':'", line); + parsed_line.port_name = Strings::trim(StringView{line.data(), line.data() + colon_loc}).to_string(); + + auto equal_loc = line.find('=', colon_loc + 1); + Checks::check_exit(VCPKG_LINE_INFO, equal_loc != std::string::npos, "Line '%s' must contain a '='", line); + parsed_line.triplet_name = + Strings::trim(StringView{line.data() + colon_loc + 1, line.data() + equal_loc}).to_string(); + + auto baseline_value = Strings::trim(StringView{line.data() + equal_loc + 1, line.data() + line.size()}); if (baseline_value == "fail") { - callback(port, triplet, CiBaselineValue::Fail); + parsed_line.state = CiBaselineState::Fail; } else if (baseline_value == "skip") { - callback(port, triplet, CiBaselineValue::Skip); + parsed_line.state = CiBaselineState::Skip; } else { Checks::exit_with_message(VCPKG_LINE_INFO, "Unknown value '%s'", baseline_value); } } + return result; } } @@ -132,7 +144,7 @@ namespace vcpkg::Commands::CI static constexpr StringLiteral OPTION_FAILURE_LOGS = "failure-logs"; static constexpr StringLiteral OPTION_XUNIT = "x-xunit"; static constexpr StringLiteral OPTION_CI_BASELINE = "ci-baseline"; - static constexpr StringLiteral OPTION_PASSING_IS_PASSING = "passing-is-passing"; + static constexpr StringLiteral OPTION_ALLOW_UNEXPECTED_PASSING = "allow-unexpected-passing"; static constexpr StringLiteral OPTION_RANDOMIZE = "x-randomize"; static constexpr StringLiteral OPTION_OUTPUT_HASHES = "output-hashes"; static constexpr StringLiteral OPTION_PARENT_HASHES = "parent-hashes"; @@ -153,7 +165,7 @@ namespace vcpkg::Commands::CI static constexpr std::array CI_SWITCHES = {{ {OPTION_DRY_RUN, "Print out plan without execution"}, {OPTION_RANDOMIZE, "Randomize the install order"}, - {OPTION_PASSING_IS_PASSING, "Indicates that 'Passing, remove from fail list' results should not be emitted."}, + {OPTION_ALLOW_UNEXPECTED_PASSING, "Indicates that 'Passing, remove from fail list' results should not be emitted."}, }}; const CommandStructure COMMAND_STRUCTURE = { @@ -530,37 +542,46 @@ namespace vcpkg::Commands::CI auto exclusions = parse_exclusions(settings, OPTION_EXCLUDE); auto host_exclusions = parse_exclusions(settings, OPTION_HOST_EXCLUDE); auto baseline_iter = settings.find(OPTION_CI_BASELINE); - const bool passing_is_passing = Util::Sets::contains(options.switches, OPTION_PASSING_IS_PASSING); + const bool allow_unexpected_passing = Util::Sets::contains(options.switches, OPTION_ALLOW_UNEXPECTED_PASSING); std::set expected_failures; if (baseline_iter != settings.end()) { - parse_ci_baseline(paths.get_filesystem().read_lines(baseline_iter->second, VCPKG_LINE_INFO), - [&, - target_triplet_string = args.command_arguments[0], - host_triplet_string = host_triplet.canonical_name()]( - StringView port, StringView triplet, CiBaselineValue value) { - if (triplet == target_triplet_string) - { - if (value == CiBaselineValue::Skip) - exclusions.insert(port.to_string()); - else if (value == CiBaselineValue::Fail) - expected_failures.emplace(port.to_string(), target_triplet); - } - else if (triplet == host_triplet_string) - { - if (value == CiBaselineValue::Skip) - host_exclusions.insert(port.to_string()); - else if (value == CiBaselineValue::Fail) - expected_failures.emplace(port.to_string(), host_triplet); - } - }); + auto baseline = + parse_ci_baseline(paths.get_filesystem().read_lines(baseline_iter->second, VCPKG_LINE_INFO)); + auto target_triplet_string = args.command_arguments[0]; + auto host_triplet_string = host_triplet.canonical_name(); + for (auto& line : baseline) + { + if (line.triplet_name == target_triplet_string) + { + if (line.state == CiBaselineState::Skip) + { + exclusions.insert(line.port_name); + } + else if (line.state == CiBaselineState::Fail) + { + expected_failures.emplace(line.port_name, target_triplet); + } + } + else if (line.triplet_name == host_triplet_string) + { + if (line.state == CiBaselineState::Skip) + { + host_exclusions.insert(line.port_name); + } + else if (line.state == CiBaselineState::Fail) + { + expected_failures.emplace(line.port_name, host_triplet); + } + } + } } else { Checks::check_exit(VCPKG_LINE_INFO, - !passing_is_passing, + !allow_unexpected_passing, Strings::concat("--", - OPTION_PASSING_IS_PASSING, + OPTION_ALLOW_UNEXPECTED_PASSING, " can only be used if a ci baseline is provided via --", OPTION_CI_BASELINE)); } @@ -777,7 +798,7 @@ namespace vcpkg::Commands::CI } break; case Build::BuildResult::SUCCEEDED: - if (!passing_is_passing && + if (!allow_unexpected_passing && expected_failures.find(port_result.spec) != expected_failures.end()) { std::cerr << " PASSING, REMOVE FROM FAIL LIST: " << port_result.spec.to_string() From 7549a57dcdd4075f6b1a9a0fe9cd23abf7d7c60f Mon Sep 17 00:00:00 2001 From: Leander Schulten Date: Sun, 6 Mar 2022 14:14:35 +0100 Subject: [PATCH 04/23] Fix build --- src/vcpkg/commands.ci.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 6ab5f9a87b..b191aa37d3 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -792,10 +792,11 @@ namespace vcpkg::Commands::CI case Build::BuildResult::FILE_CONFLICTS: if (expected_failures.find(port_result.spec) == expected_failures.end()) { - std::cerr << " REGRESSION: " << port_result.spec.to_string() << " failed with " - << Build::to_string(port_result.build_result.code) << ". If expected, add " - << port_result.spec.to_string() << "=fail to " << baseline_iter->second - << std::endl; + std::cerr + << " REGRESSION: " << port_result.spec.to_string() << " failed with " + << Build::to_string_locale_invariant(port_result.build_result.code).to_string() + << ". If expected, add " << port_result.spec.to_string() << "=fail to " + << baseline_iter->second << std::endl; } break; case Build::BuildResult::SUCCEEDED: From a3797dc38c8cf0bbe9633343bf770e2eea952e66 Mon Sep 17 00:00:00 2001 From: Leander Schulten Date: Sun, 6 Mar 2022 15:10:42 +0100 Subject: [PATCH 05/23] Fix Formatting --- src/vcpkg/commands.ci.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index b191aa37d3..807396baf7 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -165,7 +165,8 @@ namespace vcpkg::Commands::CI static constexpr std::array CI_SWITCHES = {{ {OPTION_DRY_RUN, "Print out plan without execution"}, {OPTION_RANDOMIZE, "Randomize the install order"}, - {OPTION_ALLOW_UNEXPECTED_PASSING, "Indicates that 'Passing, remove from fail list' results should not be emitted."}, + {OPTION_ALLOW_UNEXPECTED_PASSING, + "Indicates that 'Passing, remove from fail list' results should not be emitted."}, }}; const CommandStructure COMMAND_STRUCTURE = { From 8fe0d2ca0e40f27242f50e6a66456eba35d5a276 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Thu, 17 Mar 2022 23:56:28 -0700 Subject: [PATCH 06/23] WIP * Delete namespace Parse:: * Add forwarding headers for a lot of the parse machinery. * Extract some supporting structures for parsing ci.baseline.txt into a header so that tests can be added. --- include/vcpkg-test/util.h | 6 +++--- include/vcpkg/base/fwd/parse.h | 18 ++++++++++++++++++ include/vcpkg/base/json.h | 9 +++++---- include/vcpkg/base/parse.h | 10 +++------- include/vcpkg/binaryparagraph.h | 2 +- include/vcpkg/ci-baseline.h | 16 ++++++++++++++++ include/vcpkg/fwd/ci-baseline.h | 13 +++++++++++++ include/vcpkg/fwd/paragraphparser.h | 7 +++++++ include/vcpkg/packagespec.h | 13 +++++-------- include/vcpkg/paragraphparser.h | 4 +++- include/vcpkg/paragraphs.h | 16 ++++------------ include/vcpkg/sourceparagraph.h | 13 ++++++------- include/vcpkg/statusparagraph.h | 2 +- include/vcpkg/textrowcol.h | 2 +- src/vcpkg-test/manifests.cpp | 16 ++++++++-------- src/vcpkg-test/paragraph.cpp | 6 +++--- src/vcpkg-test/util.cpp | 26 +++++++++++++------------- src/vcpkg-test/versionplan.cpp | 1 - src/vcpkg/base/json.cpp | 22 +++++++++++----------- src/vcpkg/base/parse.cpp | 4 ++-- src/vcpkg/binarycaching.cpp | 6 ++---- src/vcpkg/binaryparagraph.cpp | 4 +--- src/vcpkg/build.cpp | 10 +++------- src/vcpkg/commands.ci.cpp | 14 +------------- src/vcpkg/commands.info.cpp | 4 ++-- src/vcpkg/install.cpp | 2 +- src/vcpkg/packagespec.cpp | 15 ++++++--------- src/vcpkg/paragraphs.cpp | 17 +++++++---------- src/vcpkg/platform-expression.cpp | 6 ++---- src/vcpkg/registries.cpp | 4 ++-- src/vcpkg/sourceparagraph.cpp | 24 +++++++++++------------- src/vcpkg/statusparagraph.cpp | 5 +---- src/vcpkg/userconfig.cpp | 2 +- src/vcpkg/versions.cpp | 28 ++++++++++++++-------------- 34 files changed, 177 insertions(+), 170 deletions(-) create mode 100644 include/vcpkg/base/fwd/parse.h create mode 100644 include/vcpkg/ci-baseline.h create mode 100644 include/vcpkg/fwd/ci-baseline.h create mode 100644 include/vcpkg/fwd/paragraphparser.h diff --git a/include/vcpkg-test/util.h b/include/vcpkg-test/util.h index 2f2a22238a..1e86f01a89 100644 --- a/include/vcpkg-test/util.h +++ b/include/vcpkg-test/util.h @@ -66,12 +66,12 @@ namespace vcpkg::Test inline auto test_parse_control_file(const std::vector>& v) { - std::vector pghs; + std::vector pghs; for (auto&& p : v) { pghs.emplace_back(); for (auto&& kv : p) - pghs.back().emplace(kv.first, std::make_pair(kv.second, vcpkg::Parse::TextRowCol{})); + pghs.back().emplace(kv.first, std::make_pair(kv.second, TextRowCol{})); } return vcpkg::SourceControlFile::parse_control_file("", std::move(pghs)); } @@ -126,7 +126,7 @@ namespace vcpkg::Test inline std::vector parse_test_fspecs(StringView sv, Triplet t = X86_WINDOWS) { std::vector ret; - Parse::ParserBase parser(sv, "test"); + ParserBase parser(sv, "test"); while (!parser.at_eof()) { auto opt = parse_qualified_specifier(parser); diff --git a/include/vcpkg/base/fwd/parse.h b/include/vcpkg/base/fwd/parse.h new file mode 100644 index 0000000000..d6cce6f394 --- /dev/null +++ b/include/vcpkg/base/fwd/parse.h @@ -0,0 +1,18 @@ +#pragma once + +namespace vcpkg +{ + struct IParseError; + struct ParseError; + struct SourceLoc; + + enum class MessageKind + { + Warning, + Error, + }; + + struct ParseMessage; + struct ParseMessages; + struct ParserBase; +} diff --git a/include/vcpkg/base/json.h b/include/vcpkg/base/json.h index 233b6934d1..ccb7d7eed4 100644 --- a/include/vcpkg/base/json.h +++ b/include/vcpkg/base/json.h @@ -289,10 +289,11 @@ namespace vcpkg::Json underlying_t underlying_; }; - ExpectedT, std::unique_ptr> parse_file( - const Filesystem&, const Path&, std::error_code& ec) noexcept; - ExpectedT, std::unique_ptr> parse(StringView text, - StringView origin = {}) noexcept; + ExpectedT, std::unique_ptr> parse_file(const Filesystem&, + const Path&, + std::error_code& ec) noexcept; + ExpectedT, std::unique_ptr> parse(StringView text, + StringView origin = {}) noexcept; std::pair parse_file(vcpkg::LineInfo li, const Filesystem&, const Path&) noexcept; std::string stringify(const Value&, JsonStyle style); diff --git a/include/vcpkg/base/parse.h b/include/vcpkg/base/parse.h index a9364ee905..a91fd8fd99 100644 --- a/include/vcpkg/base/parse.h +++ b/include/vcpkg/base/parse.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include #include @@ -11,7 +13,7 @@ #include #include -namespace vcpkg::Parse +namespace vcpkg { struct IParseError { @@ -51,12 +53,6 @@ namespace vcpkg::Parse int column; }; - enum class MessageKind - { - Warning, - Error, - }; - struct ParseMessage { SourceLoc location = {}; diff --git a/include/vcpkg/binaryparagraph.h b/include/vcpkg/binaryparagraph.h index 7bc4682a8a..476bc5198c 100644 --- a/include/vcpkg/binaryparagraph.h +++ b/include/vcpkg/binaryparagraph.h @@ -12,7 +12,7 @@ namespace vcpkg struct BinaryParagraph { BinaryParagraph(); - explicit BinaryParagraph(Parse::Paragraph fields); + explicit BinaryParagraph(Paragraph fields); BinaryParagraph(const SourceParagraph& spgh, Triplet triplet, const std::string& abi_tag, diff --git a/include/vcpkg/ci-baseline.h b/include/vcpkg/ci-baseline.h new file mode 100644 index 0000000000..4a6f8a4509 --- /dev/null +++ b/include/vcpkg/ci-baseline.h @@ -0,0 +1,16 @@ +#pragma once +#include + +#include + +#include + +namespace vcpkg +{ + struct CiBaselineLine + { + std::string port_name; + std::string triplet_name; + CiBaselineState state; + }; +} \ No newline at end of file diff --git a/include/vcpkg/fwd/ci-baseline.h b/include/vcpkg/fwd/ci-baseline.h new file mode 100644 index 0000000000..d4dce6d1a2 --- /dev/null +++ b/include/vcpkg/fwd/ci-baseline.h @@ -0,0 +1,13 @@ +#pragma once + +namespace vcpkg +{ + struct CiBaseline; + struct CiBaselineLine; + + enum class CiBaselineState + { + Skip, + Fail, + }; +} diff --git a/include/vcpkg/fwd/paragraphparser.h b/include/vcpkg/fwd/paragraphparser.h new file mode 100644 index 0000000000..96a50418b6 --- /dev/null +++ b/include/vcpkg/fwd/paragraphparser.h @@ -0,0 +1,7 @@ +#pragma once + +namespace vcpkg +{ + struct ParseControlErrorInfo; + struct ParagraphParser; +} diff --git a/include/vcpkg/packagespec.h b/include/vcpkg/packagespec.h index a9b9ee6cbf..30a59f1b92 100644 --- a/include/vcpkg/packagespec.h +++ b/include/vcpkg/packagespec.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include #include @@ -10,11 +12,6 @@ #include #include -namespace vcpkg::Parse -{ - struct ParserBase; -} - namespace vcpkg { /// @@ -190,10 +187,10 @@ namespace vcpkg ExpectedS to_package_spec(Triplet default_triplet) const; }; - Optional parse_feature_name(Parse::ParserBase& parser); - Optional parse_package_name(Parse::ParserBase& parser); + Optional parse_feature_name(ParserBase& parser); + Optional parse_package_name(ParserBase& parser); ExpectedS parse_qualified_specifier(StringView input); - Optional parse_qualified_specifier(Parse::ParserBase& parser); + Optional parse_qualified_specifier(ParserBase& parser); } template diff --git a/include/vcpkg/paragraphparser.h b/include/vcpkg/paragraphparser.h index f480278984..1aa246e8eb 100644 --- a/include/vcpkg/paragraphparser.h +++ b/include/vcpkg/paragraphparser.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include @@ -11,7 +13,7 @@ #include #include -namespace vcpkg::Parse +namespace vcpkg { struct ParseControlErrorInfo { diff --git a/include/vcpkg/paragraphs.h b/include/vcpkg/paragraphs.h index d007ef6bd9..5018000a7d 100644 --- a/include/vcpkg/paragraphs.h +++ b/include/vcpkg/paragraphs.h @@ -1,20 +1,14 @@ #pragma once +#include #include #include #include -namespace vckpg::Parse -{ - struct ParseControlErrorInfo; -} - namespace vcpkg::Paragraphs { - using Paragraph = Parse::Paragraph; - uint64_t get_load_ports_stats(); ExpectedS parse_single_paragraph(StringView str, StringView origin); @@ -26,10 +20,8 @@ namespace vcpkg::Paragraphs bool is_port_directory(const Filesystem& fs, const Path& maybe_directory); - Parse::ParseExpected try_load_port(const Filesystem& fs, const Path& port_directory); - Parse::ParseExpected try_load_port_text(const std::string& text, - StringView origin, - bool is_manifest); + ParseExpected try_load_port(const Filesystem& fs, const Path& port_directory); + ParseExpected try_load_port_text(const std::string& text, StringView origin, bool is_manifest); ExpectedS try_load_cached_package(const Filesystem& fs, const Path& package_dir, @@ -38,7 +30,7 @@ namespace vcpkg::Paragraphs struct LoadResults { std::vector paragraphs; - std::vector> errors; + std::vector> errors; }; // this allows one to pass this around as an overload set to stuff like `Util::fmap`, diff --git a/include/vcpkg/sourceparagraph.h b/include/vcpkg/sourceparagraph.h index 5c541bd1b1..3164c1ef62 100644 --- a/include/vcpkg/sourceparagraph.h +++ b/include/vcpkg/sourceparagraph.h @@ -101,11 +101,10 @@ namespace vcpkg { SourceControlFile clone() const; - static Parse::ParseExpected parse_manifest_object(StringView origin, - const Json::Object& object); + static ParseExpected parse_manifest_object(StringView origin, const Json::Object& object); - static Parse::ParseExpected parse_control_file( - StringView origin, std::vector&& control_paragraphs); + static ParseExpected parse_control_file(StringView origin, + std::vector&& control_paragraphs); // Always non-null in non-error cases std::unique_ptr core_paragraph; @@ -147,11 +146,11 @@ namespace vcpkg Path source_location; }; - void print_error_message(Span> error_info_list); - inline void print_error_message(const std::unique_ptr& error_info_list) + void print_error_message(Span> error_info_list); + inline void print_error_message(const std::unique_ptr& error_info_list) { return print_error_message({&error_info_list, 1}); } - std::string parse_spdx_license_expression(StringView sv, Parse::ParseMessages& messages); + std::string parse_spdx_license_expression(StringView sv, ParseMessages& messages); } diff --git a/include/vcpkg/statusparagraph.h b/include/vcpkg/statusparagraph.h index 53928c49d0..acc9355f44 100644 --- a/include/vcpkg/statusparagraph.h +++ b/include/vcpkg/statusparagraph.h @@ -34,7 +34,7 @@ namespace vcpkg struct StatusParagraph { StatusParagraph() noexcept; - explicit StatusParagraph(Parse::Paragraph&& fields); + explicit StatusParagraph(Paragraph&& fields); bool is_installed() const { return want == Want::INSTALL && state == InstallState::INSTALLED; } diff --git a/include/vcpkg/textrowcol.h b/include/vcpkg/textrowcol.h index b8269f6979..d7e216d852 100644 --- a/include/vcpkg/textrowcol.h +++ b/include/vcpkg/textrowcol.h @@ -1,6 +1,6 @@ #pragma once -namespace vcpkg::Parse +namespace vcpkg { struct TextRowCol { diff --git a/src/vcpkg-test/manifests.cpp b/src/vcpkg-test/manifests.cpp index cd041c57fe..01054df382 100644 --- a/src/vcpkg-test/manifests.cpp +++ b/src/vcpkg-test/manifests.cpp @@ -36,8 +36,8 @@ enum class PrintErrors : bool Yes, }; -static Parse::ParseExpected test_parse_manifest(const Json::Object& obj, - PrintErrors print = PrintErrors::Yes) +static ParseExpected test_parse_manifest(const Json::Object& obj, + PrintErrors print = PrintErrors::Yes) { auto res = SourceControlFile::parse_manifest_object("", obj); if (!res.has_value() && print == PrintErrors::Yes) @@ -46,7 +46,7 @@ static Parse::ParseExpected test_parse_manifest(const Json::O } return res; } -static Parse::ParseExpected test_parse_manifest(StringView obj, PrintErrors print = PrintErrors::Yes) +static ParseExpected test_parse_manifest(StringView obj, PrintErrors print = PrintErrors::Yes) { return test_parse_manifest(parse_json_object(obj), print); } @@ -983,20 +983,20 @@ static std::string test_serialized_license(StringView license) static bool license_is_parseable(StringView license) { - Parse::ParseMessages messages; + ParseMessages messages; parse_spdx_license_expression(license, messages); return messages.error == nullptr; } static bool license_is_strict(StringView license) { - Parse::ParseMessages messages; + ParseMessages messages; parse_spdx_license_expression(license, messages); return messages.error == nullptr && messages.warnings.empty(); } -static std::string test_format_parse_warning(const Parse::ParseMessage& msg) +static std::string test_format_parse_warning(const ParseMessage& msg) { - return msg.format("", Parse::MessageKind::Warning).extract_data(); + return msg.format("", MessageKind::Warning).extract_data(); } TEST_CASE ("simple license in manifest", "[manifests][license]") @@ -1046,7 +1046,7 @@ TEST_CASE ("license serialization", "[manifests][license]") TEST_CASE ("license error messages", "[manifests][license]") { - Parse::ParseMessages messages; + ParseMessages messages; parse_spdx_license_expression("", messages); REQUIRE(messages.error); CHECK(messages.error->format() == R"(:1:1: error: SPDX license expression was empty. diff --git a/src/vcpkg-test/paragraph.cpp b/src/vcpkg-test/paragraph.cpp index 0239e4b6f5..4e60b8c437 100644 --- a/src/vcpkg-test/paragraph.cpp +++ b/src/vcpkg-test/paragraph.cpp @@ -7,7 +7,7 @@ #include namespace Strings = vcpkg::Strings; -using vcpkg::Parse::Paragraph; +using vcpkg::Paragraph; namespace { @@ -18,7 +18,7 @@ namespace { pghs.emplace_back(); for (auto&& kv : p) - pghs.back().emplace(kv.first, std::make_pair(kv.second, vcpkg::Parse::TextRowCol{})); + pghs.back().emplace(kv.first, std::make_pair(kv.second, vcpkg::TextRowCol{})); } return vcpkg::SourceControlFile::parse_control_file("", std::move(pghs)); } @@ -27,7 +27,7 @@ namespace { Paragraph pgh; for (auto&& kv : v) - pgh.emplace(kv.first, std::make_pair(kv.second, vcpkg::Parse::TextRowCol{})); + pgh.emplace(kv.first, std::make_pair(kv.second, vcpkg::TextRowCol{})); return vcpkg::BinaryParagraph(std::move(pgh)); } diff --git a/src/vcpkg-test/util.cpp b/src/vcpkg-test/util.cpp index b575ee6f98..1245f46c0c 100644 --- a/src/vcpkg-test/util.cpp +++ b/src/vcpkg-test/util.cpp @@ -53,13 +53,13 @@ namespace vcpkg::Test const char* default_features, const char* triplet) { - return std::make_unique(Parse::Paragraph{{"Package", {name, {}}}, - {"Version", {"1", {}}}, - {"Architecture", {triplet, {}}}, - {"Multi-Arch", {"same", {}}}, - {"Depends", {depends, {}}}, - {"Default-Features", {default_features, {}}}, - {"Status", {"install ok installed", {}}}}); + return std::make_unique(Paragraph{{"Package", {name, {}}}, + {"Version", {"1", {}}}, + {"Architecture", {triplet, {}}}, + {"Multi-Arch", {"same", {}}}, + {"Depends", {depends, {}}}, + {"Default-Features", {default_features, {}}}, + {"Status", {"install ok installed", {}}}}); } std::unique_ptr make_status_feature_pgh(const char* name, @@ -67,12 +67,12 @@ namespace vcpkg::Test const char* depends, const char* triplet) { - return std::make_unique(Parse::Paragraph{{"Package", {name, {}}}, - {"Feature", {feature, {}}}, - {"Architecture", {triplet, {}}}, - {"Multi-Arch", {"same", {}}}, - {"Depends", {depends, {}}}, - {"Status", {"install ok installed", {}}}}); + return std::make_unique(Paragraph{{"Package", {name, {}}}, + {"Feature", {feature, {}}}, + {"Architecture", {triplet, {}}}, + {"Multi-Arch", {"same", {}}}, + {"Depends", {depends, {}}}, + {"Status", {"install ok installed", {}}}}); } PackageSpec PackageSpecMap::emplace(const char* name, diff --git a/src/vcpkg-test/versionplan.cpp b/src/vcpkg-test/versionplan.cpp index 2e165807d3..2665530a7e 100644 --- a/src/vcpkg-test/versionplan.cpp +++ b/src/vcpkg-test/versionplan.cpp @@ -9,7 +9,6 @@ #include using namespace vcpkg; -using namespace vcpkg::Parse; TEST_CASE ("parse depends", "[dependencies]") { diff --git a/src/vcpkg/base/json.cpp b/src/vcpkg/base/json.cpp index a44661e5e6..e306905a39 100644 --- a/src/vcpkg/base/json.cpp +++ b/src/vcpkg/base/json.cpp @@ -470,16 +470,16 @@ namespace vcpkg::Json // auto parse() { namespace { - struct Parser : private Parse::ParserBase + struct Parser : private ParserBase { - Parser(StringView text, StringView origin) : Parse::ParserBase(text, origin), style_() { } + Parser(StringView text, StringView origin) : ParserBase(text, origin), style_() { } char32_t next() noexcept { auto ch = cur(); if (ch == '\r') style_.newline_kind = JsonStyle::Newline::CrLf; if (ch == '\t') style_.set_tabs(); - return Parse::ParserBase::next(); + return ParserBase::next(); } static constexpr bool is_digit(char32_t code_point) noexcept @@ -990,7 +990,7 @@ namespace vcpkg::Json } } - static ExpectedT, std::unique_ptr> parse( + static ExpectedT, std::unique_ptr> parse( StringView json, StringView origin) noexcept { StatsTimer t(g_json_parsing_stats); @@ -1032,7 +1032,7 @@ namespace vcpkg::Json static constexpr bool is_lower_digit(char ch) { - return Parse::ParserBase::is_lower_alpha(ch) || Parse::ParserBase::is_ascii_digit(ch); + return ParserBase::is_lower_alpha(ch) || ParserBase::is_ascii_digit(ch); } bool IdentifierDeserializer::is_ident(StringView sv) @@ -1081,14 +1081,14 @@ namespace vcpkg::Json return true; } - ExpectedT, std::unique_ptr> parse_file(const Filesystem& fs, - const Path& json_file, - std::error_code& ec) noexcept + ExpectedT, std::unique_ptr> parse_file(const Filesystem& fs, + const Path& json_file, + std::error_code& ec) noexcept { auto res = fs.read_contents(json_file, ec); if (ec) { - return std::unique_ptr(); + return std::unique_ptr(); } return parse(std::move(res), json_file); @@ -1112,8 +1112,8 @@ namespace vcpkg::Json return ret.value_or_exit(li); } - ExpectedT, std::unique_ptr> parse(StringView json, - StringView origin) noexcept + ExpectedT, std::unique_ptr> parse(StringView json, + StringView origin) noexcept { return Parser::parse(json, origin); } diff --git a/src/vcpkg/base/parse.cpp b/src/vcpkg/base/parse.cpp index b7f32efe97..6b97cb6602 100644 --- a/src/vcpkg/base/parse.cpp +++ b/src/vcpkg/base/parse.cpp @@ -8,7 +8,7 @@ using namespace vcpkg; -namespace vcpkg::Parse +namespace vcpkg { static void advance_rowcol(char32_t ch, int& row, int& column) { @@ -64,7 +64,7 @@ namespace vcpkg::Parse res.appendnl(); - auto line_end = Util::find_if(location.it, Parse::ParserBase::is_lineend); + auto line_end = Util::find_if(location.it, ParserBase::is_lineend); StringView line = StringView{ location.start_of_line.pointer_to_current(), line_end.pointer_to_current(), diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 2cc246074c..03cc4fd72b 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -39,11 +39,9 @@ namespace "Restored {count} packages from AWS servers in {elapsed}"); DECLARE_AND_REGISTER_MESSAGE(AwsUploadedPackages, (msg::count), "", "Uploaded binaries to {count} AWS servers"); - using Parse::SourceLoc; - - struct ConfigSegmentsParser : Parse::ParserBase + struct ConfigSegmentsParser : ParserBase { - using Parse::ParserBase::ParserBase; + using ParserBase::ParserBase; void parse_segments(std::vector>& out_segments); std::vector>> parse_all_segments(); diff --git a/src/vcpkg/binaryparagraph.cpp b/src/vcpkg/binaryparagraph.cpp index 4dd380f638..4fb55c8483 100644 --- a/src/vcpkg/binaryparagraph.cpp +++ b/src/vcpkg/binaryparagraph.cpp @@ -30,10 +30,8 @@ namespace vcpkg BinaryParagraph::BinaryParagraph() = default; - BinaryParagraph::BinaryParagraph(Parse::Paragraph fields) + BinaryParagraph::BinaryParagraph(Paragraph fields) { - using namespace vcpkg::Parse; - ParagraphParser parser(std::move(fields)); { diff --git a/src/vcpkg/build.cpp b/src/vcpkg/build.cpp index 294dc8c723..21b4fd3e74 100644 --- a/src/vcpkg/build.cpp +++ b/src/vcpkg/build.cpp @@ -34,14 +34,10 @@ using namespace vcpkg; using vcpkg::Build::BuildResult; -using vcpkg::Parse::ParseControlErrorInfo; -using vcpkg::Parse::ParseExpected; using vcpkg::PortFileProvider::PathsPortFileProvider; namespace { - using vcpkg::PackageSpec; - using vcpkg::VcpkgPaths; using vcpkg::Build::IBuildLogsRecorder; struct NullBuildLogsRecorder final : IBuildLogsRecorder { @@ -1514,9 +1510,9 @@ namespace vcpkg::Build paths.get_toolver_diagnostics()); } - static BuildInfo inner_create_buildinfo(Parse::Paragraph pgh) + static BuildInfo inner_create_buildinfo(Paragraph pgh) { - Parse::ParagraphParser parser(std::move(pgh)); + ParagraphParser parser(std::move(pgh)); BuildInfo build_info; @@ -1580,7 +1576,7 @@ namespace vcpkg::Build BuildInfo read_build_info(const Filesystem& fs, const Path& filepath) { - const ExpectedS pghs = Paragraphs::get_single_paragraph(fs, filepath); + const ExpectedS pghs = Paragraphs::get_single_paragraph(fs, filepath); Checks::check_maybe_upgrade( VCPKG_LINE_INFO, pghs.get() != nullptr, "Invalid BUILD_INFO file for package: %s", pghs.error()); return inner_create_buildinfo(*pghs.get()); diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 807396baf7..0bac83c4df 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -78,19 +79,6 @@ namespace Path base_path; }; - enum class CiBaselineState - { - Skip, - Fail, - }; - - struct CiBaselineLine - { - std::string port_name; - std::string triplet_name; - CiBaselineState state; - }; - std::vector parse_ci_baseline(View lines) { std::vector result; diff --git a/src/vcpkg/commands.info.cpp b/src/vcpkg/commands.info.cpp index 7bd7a83b9d..23e54ab682 100644 --- a/src/vcpkg/commands.info.cpp +++ b/src/vcpkg/commands.info.cpp @@ -57,7 +57,7 @@ namespace vcpkg::Commands::Info std::vector specs_to_write; for (auto&& arg : args.command_arguments) { - Parse::ParserBase parser(arg, ""); + ParserBase parser(arg, ""); auto maybe_qpkg = parse_qualified_specifier(parser); if (!parser.at_eof() || !maybe_qpkg) { @@ -117,7 +117,7 @@ namespace vcpkg::Commands::Info for (auto&& arg : args.command_arguments) { - Parse::ParserBase parser(arg, ""); + ParserBase parser(arg, ""); auto maybe_pkg = parse_package_name(parser); if (!parser.at_eof() || !maybe_pkg) { diff --git a/src/vcpkg/install.cpp b/src/vcpkg/install.cpp index ee55d6719e..abe3de386e 100644 --- a/src/vcpkg/install.cpp +++ b/src/vcpkg/install.cpp @@ -712,7 +712,7 @@ namespace vcpkg::Install { static auto cmakeify = [](std::string name) { auto n = Strings::ascii_to_uppercase(Strings::replace_all(std::move(name), "-", "_")); - if (n.empty() || Parse::ParserBase::is_ascii_digit(n[0])) + if (n.empty() || ParserBase::is_ascii_digit(n[0])) { n.insert(n.begin(), '_'); } diff --git a/src/vcpkg/packagespec.cpp b/src/vcpkg/packagespec.cpp index 0f175c6c35..f5ee89a973 100644 --- a/src/vcpkg/packagespec.cpp +++ b/src/vcpkg/packagespec.cpp @@ -101,7 +101,7 @@ namespace vcpkg static bool is_package_name_char(char32_t ch) { - return Parse::ParserBase::is_lower_alpha(ch) || Parse::ParserBase::is_ascii_digit(ch) || ch == '-'; + return ParserBase::is_lower_alpha(ch) || ParserBase::is_ascii_digit(ch) || ch == '-'; } static bool is_feature_name_char(char32_t ch) @@ -114,16 +114,15 @@ namespace vcpkg ExpectedS parse_qualified_specifier(StringView input) { - auto parser = Parse::ParserBase(input, ""); + auto parser = ParserBase(input, ""); auto maybe_pqs = parse_qualified_specifier(parser); if (!parser.at_eof()) parser.add_error("expected eof"); if (auto e = parser.get_error()) return e->format(); return std::move(maybe_pqs).value_or_exit(VCPKG_LINE_INFO); } - Optional parse_feature_name(Parse::ParserBase& parser) + Optional parse_feature_name(ParserBase& parser) { - using Parse::ParserBase; auto ret = parser.match_zero_or_more(is_feature_name_char).to_string(); auto ch = parser.cur(); @@ -148,9 +147,8 @@ namespace vcpkg } return ret; } - Optional parse_package_name(Parse::ParserBase& parser) + Optional parse_package_name(ParserBase& parser) { - using Parse::ParserBase; auto ret = parser.match_zero_or_more(is_package_name_char).to_string(); auto ch = parser.cur(); if (ParserBase::is_upper_alpha(ch) || ch == '_') @@ -166,9 +164,8 @@ namespace vcpkg return ret; } - Optional parse_qualified_specifier(Parse::ParserBase& parser) + Optional parse_qualified_specifier(ParserBase& parser) { - using Parse::ParserBase; ParsedQualifiedSpecifier ret; auto name = parse_package_name(parser); if (auto n = name.get()) @@ -209,7 +206,7 @@ namespace vcpkg } else { - if (skipped_space.size() > 0 || Parse::ParserBase::is_lineend(parser.cur())) + if (skipped_space.size() > 0 || ParserBase::is_lineend(parser.cur())) parser.add_error("expected ',' or ']' in feature list"); else parser.add_error("invalid character in feature name (must be lowercase, digits, '-', or '*')"); diff --git a/src/vcpkg/paragraphs.cpp b/src/vcpkg/paragraphs.cpp index 9d97e996b8..c8af8958f8 100644 --- a/src/vcpkg/paragraphs.cpp +++ b/src/vcpkg/paragraphs.cpp @@ -10,12 +10,9 @@ #include #include -using namespace vcpkg::Parse; -using namespace vcpkg; - static std::atomic g_load_ports_stats(0); -namespace vcpkg::Parse +namespace vcpkg { static Optional> remove_field(Paragraph* fields, const std::string& fieldname) { @@ -78,7 +75,7 @@ namespace vcpkg::Parse } template - static Optional> parse_list_until_eof(StringLiteral plural_item_name, Parse::ParserBase& parser, F f) + static Optional> parse_list_until_eof(StringLiteral plural_item_name, ParserBase& parser, F f) { std::vector ret; parser.skip_whitespace(); @@ -104,7 +101,7 @@ namespace vcpkg::Parse StringView origin, TextRowCol textrowcol) { - auto parser = Parse::ParserBase(str, origin, textrowcol); + auto parser = ParserBase(str, origin, textrowcol); auto opt = parse_list_until_eof("default features", parser, &parse_feature_name); if (!opt) return {parser.get_error()->format(), expected_right_tag}; return {std::move(opt).value_or_exit(VCPKG_LINE_INFO), expected_left_tag}; @@ -113,7 +110,7 @@ namespace vcpkg::Parse StringView origin, TextRowCol textrowcol) { - auto parser = Parse::ParserBase(str, origin, textrowcol); + auto parser = ParserBase(str, origin, textrowcol); auto opt = parse_list_until_eof( "dependencies", parser, [](ParserBase& parser) { return parse_qualified_specifier(parser); }); if (!opt) return {parser.get_error()->format(), expected_right_tag}; @@ -124,7 +121,7 @@ namespace vcpkg::Parse StringView origin, TextRowCol textrowcol) { - auto parser = Parse::ParserBase(str, origin, textrowcol); + auto parser = ParserBase(str, origin, textrowcol); auto opt = parse_list_until_eof("dependencies", parser, [](ParserBase& parser) { auto loc = parser.cur_loc(); return parse_qualified_specifier(parser).then([&](ParsedQualifiedSpecifier&& pqs) -> Optional { @@ -144,7 +141,7 @@ namespace vcpkg::Parse namespace vcpkg::Paragraphs { - struct PghParser : private Parse::ParserBase + struct PghParser : private ParserBase { private: void get_fieldvalue(std::string& fieldvalue) @@ -197,7 +194,7 @@ namespace vcpkg::Paragraphs } public: - PghParser(StringView text, StringView origin) : Parse::ParserBase(text, origin) { } + PghParser(StringView text, StringView origin) : ParserBase(text, origin) { } ExpectedS> get_paragraphs() { diff --git a/src/vcpkg/platform-expression.cpp b/src/vcpkg/platform-expression.cpp index 6414dfe572..69ae0e43f2 100644 --- a/src/vcpkg/platform-expression.cpp +++ b/src/vcpkg/platform-expression.cpp @@ -11,8 +11,6 @@ namespace vcpkg::PlatformExpression { - using vcpkg::Parse::ParseError; - enum class Identifier { invalid = -1, // not a recognized identifier @@ -109,10 +107,10 @@ namespace vcpkg::PlatformExpression } }; - struct ExpressionParser : Parse::ParserBase + struct ExpressionParser : ParserBase { ExpressionParser(StringView str, MultipleBinaryOperators multiple_binary_operators) - : Parse::ParserBase(str, "CONTROL"), multiple_binary_operators(multiple_binary_operators) + : ParserBase(str, "CONTROL"), multiple_binary_operators(multiple_binary_operators) { } diff --git a/src/vcpkg/registries.cpp b/src/vcpkg/registries.cpp index 3689a51340..97dcfdc730 100644 --- a/src/vcpkg/registries.cpp +++ b/src/vcpkg/registries.cpp @@ -252,14 +252,14 @@ namespace Checks::exit_fail(li); } - const Parse::ParseExpected& get_scf(const Path& path) const + const ParseExpected& get_scf(const Path& path) const { return m_scfs.get_lazy(path, [this, &path]() { return Paragraphs::try_load_port(m_fs, path); }); } const Filesystem& m_fs; const Path m_builtin_ports_directory; - Cache> m_scfs; + Cache> m_scfs; }; constexpr StringLiteral BuiltinFilesRegistry::s_kind; diff --git a/src/vcpkg/sourceparagraph.cpp b/src/vcpkg/sourceparagraph.cpp index 872adfbdd5..5f926d5273 100644 --- a/src/vcpkg/sourceparagraph.cpp +++ b/src/vcpkg/sourceparagraph.cpp @@ -100,8 +100,6 @@ namespace namespace vcpkg { - using namespace vcpkg::Parse; - template static bool paragraph_equal(const Lhs& lhs, const Rhs& rhs) { @@ -187,7 +185,7 @@ namespace vcpkg return valid_fields; } - void print_error_message(Span> error_info_list); + void print_error_message(Span> error_info_list); std::string Type::to_string(const Type& t) { @@ -448,12 +446,12 @@ namespace vcpkg return fpgh; } - ParseExpected SourceControlFile::parse_control_file( - StringView origin, std::vector&& control_paragraphs) + ParseExpected SourceControlFile::parse_control_file(StringView origin, + std::vector&& control_paragraphs) { if (control_paragraphs.size() == 0) { - auto ret = std::make_unique(); + auto ret = std::make_unique(); ret->name = origin.to_string(); return ret; } @@ -801,9 +799,9 @@ namespace vcpkg // * a string, which must be an SPDX license expression. // EBNF located at: https://github.com/microsoft/vcpkg/blob/master/docs/maintainers/manifest-files.md#license // * `null`, for when the license of the package cannot be described by an SPDX expression - struct SpdxLicenseExpressionParser : Parse::ParserBase + struct SpdxLicenseExpressionParser : ParserBase { - SpdxLicenseExpressionParser(StringView sv, StringView origin) : Parse::ParserBase(sv, origin) { } + SpdxLicenseExpressionParser(StringView sv, StringView origin) : ParserBase(sv, origin) { } static const StringLiteral* case_insensitive_find(View lst, StringView id) { @@ -1005,7 +1003,7 @@ namespace vcpkg } }; - std::string parse_spdx_license_expression(StringView sv, Parse::ParseMessages& messages) + std::string parse_spdx_license_expression(StringView sv, ParseMessages& messages) { auto parser = SpdxLicenseExpressionParser(sv, ""); auto result = parser.parse(); @@ -1028,7 +1026,7 @@ namespace vcpkg for (const auto& warning : parser.messages().warnings) { - msg::println(Color::warning, warning.format("", Parse::MessageKind::Warning)); + msg::println(Color::warning, warning.format("", MessageKind::Warning)); } if (auto err = parser.get_error()) { @@ -1270,8 +1268,8 @@ namespace vcpkg return ret; } - Parse::ParseExpected SourceControlFile::parse_manifest_object(StringView origin, - const Json::Object& manifest) + ParseExpected SourceControlFile::parse_manifest_object(StringView origin, + const Json::Object& manifest) { Json::Reader reader; @@ -1381,7 +1379,7 @@ namespace vcpkg return nullopt; } - void print_error_message(Span> error_info_list) + void print_error_message(Span> error_info_list) { Checks::check_exit(VCPKG_LINE_INFO, error_info_list.size() > 0); diff --git a/src/vcpkg/statusparagraph.cpp b/src/vcpkg/statusparagraph.cpp index cc2d693337..11c630508f 100644 --- a/src/vcpkg/statusparagraph.cpp +++ b/src/vcpkg/statusparagraph.cpp @@ -2,8 +2,6 @@ #include -using namespace vcpkg::Parse; - namespace vcpkg { namespace BinaryParagraphRequiredField @@ -23,8 +21,7 @@ namespace vcpkg .push_back('\n'); } - StatusParagraph::StatusParagraph(Parse::Paragraph&& fields) - : want(Want::ERROR_STATE), state(InstallState::ERROR_STATE) + StatusParagraph::StatusParagraph(Paragraph&& fields) : want(Want::ERROR_STATE), state(InstallState::ERROR_STATE) { auto status_it = fields.find(BinaryParagraphRequiredField::STATUS); Checks::check_maybe_upgrade( diff --git a/src/vcpkg/userconfig.cpp b/src/vcpkg/userconfig.cpp index d274434cdc..6d37214ae9 100644 --- a/src/vcpkg/userconfig.cpp +++ b/src/vcpkg/userconfig.cpp @@ -29,7 +29,7 @@ namespace vcpkg { const auto& pghs = *p_pghs; - Parse::Paragraph keys; + Paragraph keys; if (pghs.size() > 0) keys = pghs[0]; for (size_t x = 1; x < pghs.size(); ++x) diff --git a/src/vcpkg/versions.cpp b/src/vcpkg/versions.cpp index 4e91c6fc3c..51d404f341 100644 --- a/src/vcpkg/versions.cpp +++ b/src/vcpkg/versions.cpp @@ -105,7 +105,7 @@ namespace vcpkg return nullptr; } size_t i = 1; - while (Parse::ParserBase::is_ascii_digit(s[i])) + while (ParserBase::is_ascii_digit(s[i])) { ++i; } @@ -117,18 +117,18 @@ namespace vcpkg static const char* skip_prerelease_identifier(const char* const s) { auto cur = s; - while (Parse::ParserBase::is_ascii_digit(*cur)) + while (ParserBase::is_ascii_digit(*cur)) { ++cur; } const char ch = *cur; - if (Parse::ParserBase::is_alphadash(ch)) + if (ParserBase::is_alphadash(ch)) { // matched alpha identifier do { ++cur; - } while (Parse::ParserBase::is_alphanumdash(*cur)); + } while (ParserBase::is_alphanumdash(*cur)); return cur; } if (*s == '0') @@ -196,9 +196,9 @@ namespace vcpkg for (;;) { // Require non-empty identifier element - if (!Parse::ParserBase::is_alphanumdash(*cur)) return nullopt; + if (!ParserBase::is_alphanumdash(*cur)) return nullopt; ++cur; - while (Parse::ParserBase::is_alphanumdash(*cur)) + while (ParserBase::is_alphanumdash(*cur)) { ++cur; } @@ -316,16 +316,16 @@ namespace vcpkg if (str.size() < 10) return format_invalid_date_version(str); - bool valid = Parse::ParserBase::is_ascii_digit(str[0]); - valid |= Parse::ParserBase::is_ascii_digit(str[1]); - valid |= Parse::ParserBase::is_ascii_digit(str[2]); - valid |= Parse::ParserBase::is_ascii_digit(str[3]); + bool valid = ParserBase::is_ascii_digit(str[0]); + valid |= ParserBase::is_ascii_digit(str[1]); + valid |= ParserBase::is_ascii_digit(str[2]); + valid |= ParserBase::is_ascii_digit(str[3]); valid |= str[4] != '-'; - valid |= Parse::ParserBase::is_ascii_digit(str[5]); - valid |= Parse::ParserBase::is_ascii_digit(str[6]); + valid |= ParserBase::is_ascii_digit(str[5]); + valid |= ParserBase::is_ascii_digit(str[6]); valid |= str[7] != '-'; - valid |= Parse::ParserBase::is_ascii_digit(str[8]); - valid |= Parse::ParserBase::is_ascii_digit(str[9]); + valid |= ParserBase::is_ascii_digit(str[8]); + valid |= ParserBase::is_ascii_digit(str[9]); if (!valid) return format_invalid_date_version(str); ret.version_string.assign(str.c_str(), 10); From d5f0498d9d68e327103930a23b322a9b467f8856 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Fri, 18 Mar 2022 13:45:38 -0700 Subject: [PATCH 07/23] Extend SortedVector to allow merging different SortedVectors together. Extract exclusions / ci baseline infrastructure. --- include/vcpkg/base/sortedvector.h | 77 +++++++++++++---- include/vcpkg/ci-baseline.h | 39 ++++++++- include/vcpkg/fwd/ci-baseline.h | 3 + include/vcpkg/packagespec.h | 3 +- src/vcpkg-test/update.cpp | 16 ++-- src/vcpkg/ci-baseline.cpp | 136 ++++++++++++++++++++++++++++++ src/vcpkg/commands.ci.cpp | 127 ++++------------------------ src/vcpkg/update.cpp | 4 +- src/vcpkg/visualstudio.cpp | 4 +- 9 files changed, 268 insertions(+), 141 deletions(-) create mode 100644 src/vcpkg/ci-baseline.cpp diff --git a/include/vcpkg/base/sortedvector.h b/include/vcpkg/base/sortedvector.h index 708f5c51ae..70cf2c037f 100644 --- a/include/vcpkg/base/sortedvector.h +++ b/include/vcpkg/base/sortedvector.h @@ -1,34 +1,46 @@ #pragma once #include +#include #include +#include // Add more forwarding functions to the m_data std::vector as needed. namespace vcpkg { - template + template> struct SortedVector { - using size_type = typename std::vector::size_type; - using iterator = typename std::vector::const_iterator; + using size_type = typename std::vector::size_type; + using iterator = typename std::vector::const_iterator; - SortedVector() : m_data() { } + SortedVector() : m_data(), m_comp() { } + SortedVector(const SortedVector&) = default; + SortedVector(SortedVector&&) = default; + SortedVector& operator=(const SortedVector&) = default; + SortedVector& operator=(SortedVector&&) = default; - explicit SortedVector(std::vector v) : m_data(std::move(v)) + explicit SortedVector(const std::vector& data) : m_data(data), m_comp() { sort_uniqueify(); } + explicit SortedVector(const std::vector& data, Compare comp) : m_data(data), m_comp(comp) { - if (!std::is_sorted(m_data.begin(), m_data.end())) - { - std::sort(m_data.begin(), m_data.end()); - } + sort_uniqueify(); + } + explicit SortedVector(std::vector&& data) : m_data(std::move(data)) { sort_uniqueify(); } + explicit SortedVector(std::vector&& data, Compare comp) : m_data(std::move(data)), m_comp(comp) + { + sort_uniqueify(); } - template - SortedVector(std::vector v, Compare comp) : m_data(std::move(v)) + template + explicit SortedVector(InIt first, InIt last) : m_data(first, last), m_comp() { - if (!std::is_sorted(m_data.cbegin(), m_data.cend(), comp)) - { - std::sort(m_data.begin(), m_data.end(), comp); - } + sort_uniqueify(); + } + + template + explicit SortedVector(InIt first, InIt last, Compare comp) : m_data(first, last), m_comp(comp) + { + sort_uniqueify(); } iterator begin() const { return this->m_data.cbegin(); } @@ -43,9 +55,40 @@ namespace vcpkg size_type size() const { return this->m_data.size(); } - const T& operator[](int i) const { return this->m_data[i]; } + const Ty& operator[](std::size_t i) const { return this->m_data[i]; } + + bool contains(const Ty& element) const { return std::binary_search(m_data.begin(), m_data.end(), element); } + + void append(const SortedVector& other) + { + // This could use a more efficient merge algorithm than inplace_merge with an understanding that we will + // allocate the whole result if perf becomes a problem + auto merge_point = m_data.insert(m_data.end(), other.begin(), other.end()); + std::inplace_merge(m_data.begin(), merge_point, m_data.end()); + uniqueify(); + } + + void append(SortedVector&& other) + { + auto merge_point = m_data.insert( + m_data.end(), std::make_move_iterator(other.begin()), std::make_move_iterator(other.end())); + std::inplace_merge(m_data.begin(), merge_point, m_data.end()); + uniqueify(); + } private: - std::vector m_data; + void uniqueify() { m_data.erase(std::unique(m_data.begin(), m_data.end(), m_comp), m_data.end()); } + void sort_uniqueify() + { + if (!std::is_sorted(m_data.begin(), m_data.end(), m_comp)) + { + std::sort(m_data.begin(), m_data.end(), m_comp); + } + + uniqueify(); + } + + std::vector m_data; + Compare m_comp; }; } diff --git a/include/vcpkg/ci-baseline.h b/include/vcpkg/ci-baseline.h index 4a6f8a4509..5f9d5d53b9 100644 --- a/include/vcpkg/ci-baseline.h +++ b/include/vcpkg/ci-baseline.h @@ -2,15 +2,50 @@ #include #include +#include +#include +#include +#include + +#include #include +#include namespace vcpkg { struct CiBaselineLine { std::string port_name; - std::string triplet_name; + Triplet triplet; CiBaselineState state; }; -} \ No newline at end of file + + struct TripletExclusions + { + Triplet triplet; + SortedVector exclusions; + + TripletExclusions(const Triplet& triplet); + TripletExclusions(const Triplet& triplet, SortedVector&& exclusions); + }; + + struct ExclusionsMap + { + std::vector triplets; + + void insert(Triplet triplet); + void insert(Triplet triplet, SortedVector&& exclusions); + }; + + struct ExclusionPredicate + { + const ExclusionsMap* data; + + bool operator()(const PackageSpec& spec) const; + }; + + std::vector parse_ci_baseline(View lines); + + SortedVector parse_and_apply_ci_baseline(View lines, ExclusionsMap& exclusions_map); +} diff --git a/include/vcpkg/fwd/ci-baseline.h b/include/vcpkg/fwd/ci-baseline.h index d4dce6d1a2..c40014bc77 100644 --- a/include/vcpkg/fwd/ci-baseline.h +++ b/include/vcpkg/fwd/ci-baseline.h @@ -4,6 +4,9 @@ namespace vcpkg { struct CiBaseline; struct CiBaselineLine; + struct TripletExclusions; + struct ExclusionsMap; + struct ExclusionPredicate; enum class CiBaselineState { diff --git a/include/vcpkg/packagespec.h b/include/vcpkg/packagespec.h index 30a59f1b92..cc23fade88 100644 --- a/include/vcpkg/packagespec.h +++ b/include/vcpkg/packagespec.h @@ -23,7 +23,8 @@ namespace vcpkg struct PackageSpec { PackageSpec() = default; - PackageSpec(std::string name, Triplet triplet) : m_name(std::move(name)), m_triplet(triplet) { } + PackageSpec(const std::string& name, Triplet triplet) : m_name(name), m_triplet(triplet) { } + PackageSpec(std::string&& name, Triplet triplet) : m_name(std::move(name)), m_triplet(triplet) { } const std::string& name() const; diff --git a/src/vcpkg-test/update.cpp b/src/vcpkg-test/update.cpp index 5db1125994..c3561b6bb5 100644 --- a/src/vcpkg-test/update.cpp +++ b/src/vcpkg-test/update.cpp @@ -27,8 +27,8 @@ TEST_CASE ("find outdated packages basic", "[update]") map.emplace("a", SourceControlFileAndLocation{std::move(scf), ""}); PortFileProvider::MapPortFileProvider provider(map); - auto pkgs = SortedVector(Update::find_outdated_packages(provider, status_db), - &OutdatedPackage::compare_by_name); + auto pkgs = SortedVector( + Update::find_outdated_packages(provider, status_db), &OutdatedPackage::compare_by_name); REQUIRE(pkgs.size() == 1); REQUIRE(pkgs[0].version_diff.left.to_string() == "2"); @@ -51,8 +51,8 @@ TEST_CASE ("find outdated packages features", "[update]") map.emplace("a", SourceControlFileAndLocation{std::move(scf), ""}); PortFileProvider::MapPortFileProvider provider(map); - auto pkgs = SortedVector(Update::find_outdated_packages(provider, status_db), - &OutdatedPackage::compare_by_name); + auto pkgs = SortedVector( + Update::find_outdated_packages(provider, status_db), &OutdatedPackage::compare_by_name); REQUIRE(pkgs.size() == 1); REQUIRE(pkgs[0].version_diff.left.to_string() == "2"); @@ -77,8 +77,8 @@ TEST_CASE ("find outdated packages features 2", "[update]") map.emplace("a", SourceControlFileAndLocation{std::move(scf), ""}); PortFileProvider::MapPortFileProvider provider(map); - auto pkgs = SortedVector(Update::find_outdated_packages(provider, status_db), - &OutdatedPackage::compare_by_name); + auto pkgs = SortedVector( + Update::find_outdated_packages(provider, status_db), &OutdatedPackage::compare_by_name); REQUIRE(pkgs.size() == 1); REQUIRE(pkgs[0].version_diff.left.to_string() == "2"); @@ -98,8 +98,8 @@ TEST_CASE ("find outdated packages none", "[update]") map.emplace("a", SourceControlFileAndLocation{std::move(scf), ""}); PortFileProvider::MapPortFileProvider provider(map); - auto pkgs = SortedVector(Update::find_outdated_packages(provider, status_db), - &OutdatedPackage::compare_by_name); + auto pkgs = SortedVector( + Update::find_outdated_packages(provider, status_db), &OutdatedPackage::compare_by_name); REQUIRE(pkgs.size() == 0); } diff --git a/src/vcpkg/ci-baseline.cpp b/src/vcpkg/ci-baseline.cpp new file mode 100644 index 0000000000..48508a7747 --- /dev/null +++ b/src/vcpkg/ci-baseline.cpp @@ -0,0 +1,136 @@ +#include +#include + +#include + +#include + +using namespace vcpkg; + +namespace +{ + struct CiBaselineParser : ParserBase + { + CiBaselineParser(StringView text, StringView origin) : ParserBase(text, origin) { } + }; +} + +namespace vcpkg +{ + TripletExclusions::TripletExclusions(const Triplet& triplet) : triplet(triplet), exclusions() { } + + TripletExclusions::TripletExclusions(const Triplet& triplet, SortedVector&& exclusions) + : triplet(triplet), exclusions(std::move(exclusions)) + { + } + + void ExclusionsMap::insert(Triplet triplet) + { + for (auto& triplet_exclusions : triplets) + { + if (triplet_exclusions.triplet == triplet) + { + return; + } + } + + triplets.emplace_back(triplet); + } + + void ExclusionsMap::insert(Triplet triplet, SortedVector&& exclusions) + { + for (auto& triplet_exclusions : triplets) + { + if (triplet_exclusions.triplet == triplet) + { + triplet_exclusions.exclusions.append(std::move(exclusions)); + return; + } + } + + triplets.emplace_back(triplet, std::move(exclusions)); + } + + bool ExclusionPredicate::operator()(const PackageSpec& spec) const + { + for (const auto& triplet_exclusions : data->triplets) + { + if (triplet_exclusions.triplet == spec.triplet()) + { + return triplet_exclusions.exclusions.contains(spec.name()); + } + } + + return false; + } + + std::vector parse_ci_baseline(View lines) + { + std::vector result; + for (auto& line : lines) + { + if (line.empty() || line[0] == '#') continue; + CiBaselineLine parsed_line; + + auto colon_loc = line.find(':'); + Checks::check_exit(VCPKG_LINE_INFO, colon_loc != std::string::npos, "Line '%s' must contain a ':'", line); + parsed_line.port_name = Strings::trim(StringView{line.data(), line.data() + colon_loc}).to_string(); + + auto equal_loc = line.find('=', colon_loc + 1); + Checks::check_exit(VCPKG_LINE_INFO, equal_loc != std::string::npos, "Line '%s' must contain a '='", line); + parsed_line.triplet = Triplet::from_canonical_name( + Strings::trim(StringView{line.data() + colon_loc + 1, line.data() + equal_loc}).to_string()); + + auto baseline_value = Strings::trim(StringView{line.data() + equal_loc + 1, line.data() + line.size()}); + if (baseline_value == "fail") + { + parsed_line.state = CiBaselineState::Fail; + } + else if (baseline_value == "skip") + { + parsed_line.state = CiBaselineState::Skip; + } + else + { + Checks::exit_with_message(VCPKG_LINE_INFO, "Unknown value '%s'", baseline_value); + } + } + return result; + } + + SortedVector parse_and_apply_ci_baseline(View lines, ExclusionsMap& exclusions_map) + { + std::vector expected_failures; + std::map> added_known_fails; + for (const auto& triplet_entry : exclusions_map.triplets) + { + added_known_fails.emplace( + std::piecewise_construct, std::forward_as_tuple(triplet_entry.triplet), std::tuple<>{}); + } + + auto baseline_lines = parse_ci_baseline(lines); + for (auto& baseline_line : baseline_lines) + { + auto triplet_match = added_known_fails.find(baseline_line.triplet); + if (triplet_match != added_known_fails.end()) + { + if (baseline_line.state == CiBaselineState::Skip) + { + triplet_match->second.push_back(std::move(baseline_line.port_name)); + } + else if (baseline_line.state == CiBaselineState::Fail) + { + expected_failures.emplace_back(std::move(baseline_line.port_name), baseline_line.triplet); + } + } + } + + for (auto& triplet_entry : exclusions_map.triplets) + { + triplet_entry.exclusions.append( + SortedVector(std::move(added_known_fails.find(triplet_entry.triplet)->second))); + } + + return SortedVector{std::move(expected_failures)}; + } +} diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 0bac83c4df..90e7849395 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -78,40 +79,6 @@ namespace private: Path base_path; }; - - std::vector parse_ci_baseline(View lines) - { - std::vector result; - for (auto& line : lines) - { - if (line.empty() || line[0] == '#') continue; - CiBaselineLine parsed_line; - - auto colon_loc = line.find(':'); - Checks::check_exit(VCPKG_LINE_INFO, colon_loc != std::string::npos, "Line '%s' must contain a ':'", line); - parsed_line.port_name = Strings::trim(StringView{line.data(), line.data() + colon_loc}).to_string(); - - auto equal_loc = line.find('=', colon_loc + 1); - Checks::check_exit(VCPKG_LINE_INFO, equal_loc != std::string::npos, "Line '%s' must contain a '='", line); - parsed_line.triplet_name = - Strings::trim(StringView{line.data() + colon_loc + 1, line.data() + equal_loc}).to_string(); - - auto baseline_value = Strings::trim(StringView{line.data() + equal_loc + 1, line.data() + line.size()}); - if (baseline_value == "fail") - { - parsed_line.state = CiBaselineState::Fail; - } - else if (baseline_value == "skip") - { - parsed_line.state = CiBaselineState::Skip; - } - else - { - Checks::exit_with_message(VCPKG_LINE_INFO, "Unknown value '%s'", baseline_value); - } - } - return result; - } } namespace vcpkg::Commands::CI @@ -345,28 +312,6 @@ namespace vcpkg::Commands::CI return supports_expression.evaluate(context); } - struct ExclusionPredicate - { - std::set exclusions; - std::set host_exclusions; - Triplet target_triplet; - Triplet host_triplet; - - bool operator()(const PackageSpec& spec) const - { - bool excluded = false; - if (spec.triplet() == host_triplet) - { - excluded = excluded || Util::Sets::contains(host_exclusions, spec.name()); - } - if (spec.triplet() == target_triplet) - { - excluded = excluded || Util::Sets::contains(exclusions, spec.name()); - } - return excluded; - } - }; - static Dependencies::ActionPlan compute_full_plan(const VcpkgPaths& paths, const PortFileProvider::PortFileProvider& provider, const CMakeVars::CMakeVarProvider& var_provider, @@ -397,7 +342,7 @@ namespace vcpkg::Commands::CI } static std::unique_ptr compute_action_statuses( - const ExclusionPredicate& is_excluded, + ExclusionPredicate is_excluded, const CMakeVars::CMakeVarProvider& var_provider, const std::vector& precheck_results, const Dependencies::ActionPlan& action_plan) @@ -486,19 +431,16 @@ namespace vcpkg::Commands::CI }); } - static std::set parse_exclusions(const std::unordered_map& settings, - StringLiteral opt) + static void parse_exclusions(const std::unordered_map& settings, + StringLiteral opt, + Triplet triplet, + ExclusionsMap& exclusions_map) { - std::set exclusions_set; auto it_exclusions = settings.find(opt); if (it_exclusions != settings.end()) { - auto exclusions = Strings::split(it_exclusions->second, ','); - exclusions_set.insert(std::make_move_iterator(exclusions.begin()), - std::make_move_iterator(exclusions.end())); + exclusions_map.insert(triplet, SortedVector(Strings::split(it_exclusions->second, ','))); } - - return exclusions_set; } static Optional parse_skipped_cascade_count(const std::unordered_map& settings) @@ -529,42 +471,16 @@ namespace vcpkg::Commands::CI BinaryCache binary_cache{args, paths}; Triplet target_triplet = Triplet::from_canonical_name(args.command_arguments[0]); - auto exclusions = parse_exclusions(settings, OPTION_EXCLUDE); - auto host_exclusions = parse_exclusions(settings, OPTION_HOST_EXCLUDE); + ExclusionsMap exclusions_map; + parse_exclusions(settings, OPTION_EXCLUDE, target_triplet, exclusions_map); + parse_exclusions(settings, OPTION_HOST_EXCLUDE, host_triplet, exclusions_map); auto baseline_iter = settings.find(OPTION_CI_BASELINE); const bool allow_unexpected_passing = Util::Sets::contains(options.switches, OPTION_ALLOW_UNEXPECTED_PASSING); - std::set expected_failures; + SortedVector expected_failures; if (baseline_iter != settings.end()) { - auto baseline = - parse_ci_baseline(paths.get_filesystem().read_lines(baseline_iter->second, VCPKG_LINE_INFO)); - auto target_triplet_string = args.command_arguments[0]; - auto host_triplet_string = host_triplet.canonical_name(); - for (auto& line : baseline) - { - if (line.triplet_name == target_triplet_string) - { - if (line.state == CiBaselineState::Skip) - { - exclusions.insert(line.port_name); - } - else if (line.state == CiBaselineState::Fail) - { - expected_failures.emplace(line.port_name, target_triplet); - } - } - else if (line.triplet_name == host_triplet_string) - { - if (line.state == CiBaselineState::Skip) - { - host_exclusions.insert(line.port_name); - } - else if (line.state == CiBaselineState::Fail) - { - expected_failures.emplace(line.port_name, host_triplet); - } - } - } + expected_failures = parse_and_apply_ci_baseline( + paths.get_filesystem().read_lines(baseline_iter->second, VCPKG_LINE_INFO), exclusions_map); } else { @@ -575,12 +491,7 @@ namespace vcpkg::Commands::CI " can only be used if a ci baseline is provided via --", OPTION_CI_BASELINE)); } - ExclusionPredicate is_excluded{ - exclusions, - host_exclusions, - target_triplet, - host_triplet, - }; + auto skipped_cascade_count = parse_skipped_cascade_count(settings); const auto is_dry_run = Util::Sets::contains(options.switches, OPTION_DRY_RUN); @@ -612,8 +523,6 @@ namespace vcpkg::Commands::CI std::vector results; auto timer = ElapsedTimer::create_started(); - Input::check_triplet(target_triplet, paths); - xunitTestResults.push_collection(target_triplet.canonical_name()); std::vector all_port_names = @@ -649,7 +558,8 @@ namespace vcpkg::Commands::CI auto action_plan = compute_full_plan(paths, provider, var_provider, all_default_full_specs, serialize_options); const auto precheck_results = binary_cache.precheck(action_plan.install_actions); - auto split_specs = compute_action_statuses(is_excluded, var_provider, precheck_results, action_plan); + auto split_specs = + compute_action_statuses(ExclusionPredicate{&exclusions_map}, var_provider, precheck_results, action_plan); { std::string msg; @@ -779,7 +689,7 @@ namespace vcpkg::Commands::CI case Build::BuildResult::BUILD_FAILED: case Build::BuildResult::POST_BUILD_CHECKS_FAILED: case Build::BuildResult::FILE_CONFLICTS: - if (expected_failures.find(port_result.spec) == expected_failures.end()) + if (!expected_failures.contains(port_result.spec)) { std::cerr << " REGRESSION: " << port_result.spec.to_string() << " failed with " @@ -789,8 +699,7 @@ namespace vcpkg::Commands::CI } break; case Build::BuildResult::SUCCEEDED: - if (!allow_unexpected_passing && - expected_failures.find(port_result.spec) != expected_failures.end()) + if (!allow_unexpected_passing && expected_failures.contains(port_result.spec)) { std::cerr << " PASSING, REMOVE FROM FAIL LIST: " << port_result.spec.to_string() << " (" << baseline_iter->second << ")" << std::endl; diff --git a/src/vcpkg/update.cpp b/src/vcpkg/update.cpp index 0b2c179645..180abc7e9b 100644 --- a/src/vcpkg/update.cpp +++ b/src/vcpkg/update.cpp @@ -70,8 +70,8 @@ namespace vcpkg::Update PortFileProvider::PathsPortFileProvider provider(paths, args.overlay_ports); - const auto outdated_packages = SortedVector(find_outdated_packages(provider, status_db), - &OutdatedPackage::compare_by_name); + const auto outdated_packages = SortedVector( + find_outdated_packages(provider, status_db), &OutdatedPackage::compare_by_name); if (outdated_packages.empty()) { diff --git a/src/vcpkg/visualstudio.cpp b/src/vcpkg/visualstudio.cpp index fb6da70fd8..c9e8a70cc7 100644 --- a/src/vcpkg/visualstudio.cpp +++ b/src/vcpkg/visualstudio.cpp @@ -210,8 +210,8 @@ namespace vcpkg::VisualStudio std::vector& paths_examined = ret.paths_examined; std::vector& found_toolsets = ret.toolsets; - const SortedVector sorted{get_visual_studio_instances_internal(fs), - VisualStudioInstance::preferred_first_comparator}; + const SortedVector sorted{ + get_visual_studio_instances_internal(fs), VisualStudioInstance::preferred_first_comparator}; const bool v140_is_available = Util::find_if(sorted, [&](const VisualStudioInstance& vs_instance) { return vs_instance.major_version() == "14"; From 81c0ccb9220e88e2ee03200f5dc112939be54230 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Fri, 18 Mar 2022 16:30:06 -0700 Subject: [PATCH 08/23] Move some parsing stuff into the cpp, add ParseMessages::exit_if_errors_or_warnings, merge match_zero_or_more and match_until as merge_while, delocalize locale-invariant FormattedParseMessageLocation, start implementing actual parsing with ParserBase. --- include/vcpkg/base/parse.h | 47 ++++++++-------- include/vcpkg/ci-baseline.h | 4 +- src/vcpkg/base/parse.cpp | 60 ++++++++++++++++----- src/vcpkg/binarycaching.cpp | 2 +- src/vcpkg/ci-baseline.cpp | 89 +++++++++++++++++-------------- src/vcpkg/commands.ci.cpp | 9 +++- src/vcpkg/packagespec.cpp | 19 ++----- src/vcpkg/paragraphs.cpp | 6 +-- src/vcpkg/platform-expression.cpp | 8 +-- src/vcpkg/sourceparagraph.cpp | 4 +- 10 files changed, 140 insertions(+), 108 deletions(-) diff --git a/include/vcpkg/base/parse.h b/include/vcpkg/base/parse.h index ac6038abea..a23c8ab2c9 100644 --- a/include/vcpkg/base/parse.h +++ b/include/vcpkg/base/parse.h @@ -65,6 +65,8 @@ namespace vcpkg { std::unique_ptr error; std::vector warnings; + + void exit_if_errors_or_warnings(StringView origin) const; }; struct ParserBase @@ -77,6 +79,7 @@ namespace vcpkg static constexpr bool is_icase_alpha(char32_t ch) { return is_lower_alpha(ch) || is_upper_alpha(ch); } static constexpr bool is_ascii_digit(char32_t ch) { return ch >= '0' && ch <= '9'; } static constexpr bool is_lineend(char32_t ch) { return ch == '\r' || ch == '\n' || ch == Unicode::end_of_file; } + static constexpr bool is_not_lineend(char32_t ch) { return !is_lineend(ch); } static constexpr bool is_alphanum(char32_t ch) { return is_icase_alpha(ch) || is_ascii_digit(ch); } static constexpr bool is_alphadash(char32_t ch) { return is_icase_alpha(ch) || ch == '-'; } static constexpr bool is_alphanumdash(char32_t ch) { return is_alphanum(ch) || ch == '-'; } @@ -87,39 +90,35 @@ namespace vcpkg } static constexpr bool is_word_char(char32_t ch) { return is_alphanum(ch) || ch == '_'; } - StringView skip_whitespace() { return match_zero_or_more(is_whitespace); } - StringView skip_tabs_spaces() - { - return match_zero_or_more([](char32_t ch) { return ch == ' ' || ch == '\t'; }); - } - void skip_to_eof() { m_it = m_it.end(); } - void skip_newline() + static constexpr bool is_package_name_char(char32_t ch) { - if (cur() == '\r') next(); - if (cur() == '\n') next(); + return is_lower_alpha(ch) || is_ascii_digit(ch) || ch == '-'; } - void skip_line() + + static constexpr bool is_feature_name_char(char32_t ch) { - match_until(is_lineend); - skip_newline(); + // TODO: we do not intend underscores to be valid, however there is currently a feature using them + // (libwebp[vwebp_sdl]). + // TODO: we need to rename this feature, then remove underscores from this list. + return is_package_name_char(ch) || ch == '_'; } + StringView skip_whitespace(); + StringView skip_tabs_spaces(); + void skip_to_eof(); + void skip_newline(); + void skip_line(); + template - StringView match_zero_or_more(Pred p) + StringView match_while(Pred p) { const char* start = m_it.pointer_to_current(); auto ch = cur(); while (ch != Unicode::end_of_file && p(ch)) + { ch = next(); - return {start, m_it.pointer_to_current()}; - } - template - StringView match_until(Pred p) - { - const char* start = m_it.pointer_to_current(); - auto ch = cur(); - while (ch != Unicode::end_of_file && !p(ch)) - ch = next(); + } + return {start, m_it.pointer_to_current()}; } @@ -140,10 +139,10 @@ namespace vcpkg void add_warning(LocalizedString&& message, const SourceLoc& loc); const IParseError* get_error() const { return m_messages.error.get(); } - std::unique_ptr extract_error() { return std::move(m_messages.error); } + std::unique_ptr extract_error() && { return std::move(m_messages.error); } const ParseMessages& messages() const { return m_messages; } - ParseMessages extract_messages() { return std::move(m_messages); } + ParseMessages extract_messages() && { return std::move(m_messages); } private: Unicode::Utf8Decoder m_it; diff --git a/include/vcpkg/ci-baseline.h b/include/vcpkg/ci-baseline.h index 5f9d5d53b9..e2c3ac271c 100644 --- a/include/vcpkg/ci-baseline.h +++ b/include/vcpkg/ci-baseline.h @@ -45,7 +45,7 @@ namespace vcpkg bool operator()(const PackageSpec& spec) const; }; - std::vector parse_ci_baseline(View lines); + std::vector parse_ci_baseline(StringView text, StringView origin, ParseMessages& messages); - SortedVector parse_and_apply_ci_baseline(View lines, ExclusionsMap& exclusions_map); + SortedVector parse_and_apply_ci_baseline(View lines, ExclusionsMap& exclusions_map); } diff --git a/src/vcpkg/base/parse.cpp b/src/vcpkg/base/parse.cpp index 6b97cb6602..2ce36cb1f1 100644 --- a/src/vcpkg/base/parse.cpp +++ b/src/vcpkg/base/parse.cpp @@ -8,6 +8,16 @@ using namespace vcpkg; +namespace +{ + DECLARE_AND_REGISTER_MESSAGE(WarningsTreatedAsErrors, (), "", "previous warnings being interpreted as errors"); + + DECLARE_AND_REGISTER_MESSAGE(FormattedParseMessageExpression, + (msg::value), + "Example of {value} is 'x64 & windows'", + " on expression: {value}"); +} + namespace vcpkg { static void advance_rowcol(char32_t ch, int& row, int& column) @@ -37,21 +47,10 @@ namespace vcpkg return res; } - DECLARE_AND_REGISTER_MESSAGE(FormattedParseMessageLocation, - (msg::path, msg::row, msg::column), - "{Locked}", - "{path}:{row}:{column}: "); - DECLARE_AND_REGISTER_MESSAGE(FormattedParseMessageExpression, - (msg::value), - "Example of {value} is 'x64 & windows'", - " on expression: {value}"); - LocalizedString ParseMessage::format(StringView origin, MessageKind kind) const { - LocalizedString res = msg::format(msgFormattedParseMessageLocation, - msg::path = origin, - msg::row = location.row, - msg::column = location.column); + LocalizedString res = + LocalizedString::from_raw(fmt::format("{}:{}:{}: ", origin, location.row, location.column)); if (kind == MessageKind::Warning) { res.append(msg::format(msg::msgWarningMessage)); @@ -95,6 +94,23 @@ namespace vcpkg const std::string& ParseError::get_message() const { return this->message; } + void ParseMessages::exit_if_errors_or_warnings(StringView origin) const + { + for (const auto& warning : warnings) + { + msg::println(warning.format(origin, MessageKind::Warning)); + } + + if (error) + { + Checks::msg_exit_with_message(VCPKG_LINE_INFO, LocalizedString::from_raw(error->format())); + } + + Checks::msg_check_exit(VCPKG_LINE_INFO, + warnings.empty(), + msg::format(msg::msgErrorMessage).append(msg::format(msgWarningsTreatedAsErrors))); + } + ParserBase::ParserBase(StringView text, StringView origin, TextRowCol init_rowcol) : m_it(text.begin(), text.end()) , m_start_of_line(m_it) @@ -105,6 +121,24 @@ namespace vcpkg { } + StringView ParserBase::skip_whitespace() { return match_while(is_whitespace); } + StringView ParserBase::skip_tabs_spaces() + { + return match_while([](char32_t ch) { return ch == ' ' || ch == '\t'; }); + } + + void ParserBase::skip_to_eof() { m_it = m_it.end(); } + void ParserBase::skip_newline() + { + if (cur() == '\r') next(); + if (cur() == '\n') next(); + } + void ParserBase::skip_line() + { + match_while(is_not_lineend); + skip_newline(); + } + char32_t ParserBase::next() { if (m_it == m_it.end()) diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index e2684d5f30..4feaf94684 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -90,7 +90,7 @@ namespace std::string segment; for (;;) { - auto n = match_until([](char32_t ch) { return ch == ',' || ch == '`' || ch == ';'; }); + auto n = match_while([](char32_t ch) { return ch != ',' && ch != '`' && ch != ';'; }); Strings::append(segment, n); auto ch = cur(); if (ch == Unicode::end_of_file || ch == ',' || ch == ';') diff --git a/src/vcpkg/ci-baseline.cpp b/src/vcpkg/ci-baseline.cpp index 48508a7747..7520789aef 100644 --- a/src/vcpkg/ci-baseline.cpp +++ b/src/vcpkg/ci-baseline.cpp @@ -7,14 +7,6 @@ using namespace vcpkg; -namespace -{ - struct CiBaselineParser : ParserBase - { - CiBaselineParser(StringView text, StringView origin) : ParserBase(text, origin) { } - }; -} - namespace vcpkg { TripletExclusions::TripletExclusions(const Triplet& triplet) : triplet(triplet), exclusions() { } @@ -64,41 +56,57 @@ namespace vcpkg return false; } - std::vector parse_ci_baseline(View lines) + std::vector parse_ci_baseline(StringView text, StringView origin, ParseMessages& messages) { std::vector result; - for (auto& line : lines) + ParserBase parser(text, origin); + while (!parser.at_eof()) { - if (line.empty() || line[0] == '#') continue; - CiBaselineLine parsed_line; - - auto colon_loc = line.find(':'); - Checks::check_exit(VCPKG_LINE_INFO, colon_loc != std::string::npos, "Line '%s' must contain a ':'", line); - parsed_line.port_name = Strings::trim(StringView{line.data(), line.data() + colon_loc}).to_string(); - - auto equal_loc = line.find('=', colon_loc + 1); - Checks::check_exit(VCPKG_LINE_INFO, equal_loc != std::string::npos, "Line '%s' must contain a '='", line); - parsed_line.triplet = Triplet::from_canonical_name( - Strings::trim(StringView{line.data() + colon_loc + 1, line.data() + equal_loc}).to_string()); - - auto baseline_value = Strings::trim(StringView{line.data() + equal_loc + 1, line.data() + line.size()}); - if (baseline_value == "fail") - { - parsed_line.state = CiBaselineState::Fail; - } - else if (baseline_value == "skip") + parser.skip_whitespace(); + if (parser.cur() == '#') { - parsed_line.state = CiBaselineState::Skip; - } - else - { - Checks::exit_with_message(VCPKG_LINE_INFO, "Unknown value '%s'", baseline_value); + parser.skip_line(); + continue; } + + // port-name:triplet=(fail|skip) + auto port = parser.match_while(ParserBase::is_package_name_char); + (void)port; } + + messages = std::move(parser).extract_messages(); + //for (auto& line : lines) + //{ + // if (line.empty() || line[0] == '#') continue; + // CiBaselineLine parsed_line; + + // auto colon_loc = line.find(':'); + // Checks::check_exit(VCPKG_LINE_INFO, colon_loc != std::string::npos, "Line '%s' must contain a ':'", line); + // parsed_line.port_name = Strings::trim(StringView{line.data(), line.data() + colon_loc}).to_string(); + + // auto equal_loc = line.find('=', colon_loc + 1); + // Checks::check_exit(VCPKG_LINE_INFO, equal_loc != std::string::npos, "Line '%s' must contain a '='", line); + // parsed_line.triplet = Triplet::from_canonical_name( + // Strings::trim(StringView{line.data() + colon_loc + 1, line.data() + equal_loc}).to_string()); + + // auto baseline_value = Strings::trim(StringView{line.data() + equal_loc + 1, line.data() + line.size()}); + // if (baseline_value == "fail") + // { + // parsed_line.state = CiBaselineState::Fail; + // } + // else if (baseline_value == "skip") + // { + // parsed_line.state = CiBaselineState::Skip; + // } + // else + // { + // Checks::exit_with_message(VCPKG_LINE_INFO, "Unknown value '%s'", baseline_value); + // } + //} return result; } - SortedVector parse_and_apply_ci_baseline(View lines, ExclusionsMap& exclusions_map) + SortedVector parse_and_apply_ci_baseline(View lines, ExclusionsMap& exclusions_map) { std::vector expected_failures; std::map> added_known_fails; @@ -108,19 +116,18 @@ namespace vcpkg std::piecewise_construct, std::forward_as_tuple(triplet_entry.triplet), std::tuple<>{}); } - auto baseline_lines = parse_ci_baseline(lines); - for (auto& baseline_line : baseline_lines) + for (auto& line : lines) { - auto triplet_match = added_known_fails.find(baseline_line.triplet); + auto triplet_match = added_known_fails.find(line.triplet); if (triplet_match != added_known_fails.end()) { - if (baseline_line.state == CiBaselineState::Skip) + if (line.state == CiBaselineState::Skip) { - triplet_match->second.push_back(std::move(baseline_line.port_name)); + triplet_match->second.push_back(line.port_name); } - else if (baseline_line.state == CiBaselineState::Fail) + else if (line.state == CiBaselineState::Fail) { - expected_failures.emplace_back(std::move(baseline_line.port_name), baseline_line.triplet); + expected_failures.emplace_back(line.port_name, line.triplet); } } } diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 863096bece..65c2b624aa 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -481,8 +481,13 @@ namespace vcpkg::Commands::CI SortedVector expected_failures; if (baseline_iter != settings.end()) { - expected_failures = parse_and_apply_ci_baseline( - paths.get_filesystem().read_lines(baseline_iter->second, VCPKG_LINE_INFO), exclusions_map); + const auto& ci_baseline_file_name = baseline_iter->second; + const auto ci_baseline_file_contents = + paths.get_filesystem().read_contents(ci_baseline_file_name, VCPKG_LINE_INFO); + ParseMessages ci_parse_messages; + const auto lines = parse_ci_baseline(ci_baseline_file_contents, ci_baseline_file_name, ci_parse_messages); + ci_parse_messages.exit_if_errors_or_warnings(ci_baseline_file_name); + expected_failures = parse_and_apply_ci_baseline(lines, exclusions_map); } else { diff --git a/src/vcpkg/packagespec.cpp b/src/vcpkg/packagespec.cpp index f5ee89a973..65966466cc 100644 --- a/src/vcpkg/packagespec.cpp +++ b/src/vcpkg/packagespec.cpp @@ -99,19 +99,6 @@ namespace vcpkg return PackageSpec{name, t}; } - static bool is_package_name_char(char32_t ch) - { - return ParserBase::is_lower_alpha(ch) || ParserBase::is_ascii_digit(ch) || ch == '-'; - } - - static bool is_feature_name_char(char32_t ch) - { - // TODO: we do not intend underscores to be valid, however there is currently a feature using them - // (libwebp[vwebp_sdl]). - // TODO: we need to rename this feature, then remove underscores from this list. - return is_package_name_char(ch) || ch == '_'; - } - ExpectedS parse_qualified_specifier(StringView input) { auto parser = ParserBase(input, ""); @@ -123,7 +110,7 @@ namespace vcpkg Optional parse_feature_name(ParserBase& parser) { - auto ret = parser.match_zero_or_more(is_feature_name_char).to_string(); + auto ret = parser.match_while(ParserBase::is_feature_name_char).to_string(); auto ch = parser.cur(); // ignores the feature name vwebp_sdl as a back-compat thing @@ -149,7 +136,7 @@ namespace vcpkg } Optional parse_package_name(ParserBase& parser) { - auto ret = parser.match_zero_or_more(is_package_name_char).to_string(); + auto ret = parser.match_while(ParserBase::is_package_name_char).to_string(); auto ch = parser.cur(); if (ParserBase::is_upper_alpha(ch) || ch == '_') { @@ -218,7 +205,7 @@ namespace vcpkg if (ch == ':') { parser.next(); - ret.triplet = parser.match_zero_or_more(is_package_name_char).to_string(); + ret.triplet = parser.match_while(ParserBase::is_package_name_char).to_string(); if (ret.triplet.get()->empty()) { parser.add_error("expected triplet name (must be lowercase, digits, '-')"); diff --git a/src/vcpkg/paragraphs.cpp b/src/vcpkg/paragraphs.cpp index c8af8958f8..7ee7781aea 100644 --- a/src/vcpkg/paragraphs.cpp +++ b/src/vcpkg/paragraphs.cpp @@ -151,7 +151,7 @@ namespace vcpkg::Paragraphs do { // scan to end of current line (it is part of the field value) - Strings::append(fieldvalue, match_until(is_lineend)); + Strings::append(fieldvalue, match_while(is_not_lineend)); skip_newline(); if (cur() != ' ') return; @@ -163,7 +163,7 @@ namespace vcpkg::Paragraphs void get_fieldname(std::string& fieldname) { - fieldname = match_zero_or_more(is_alphanumdash).to_string(); + fieldname = match_while(is_alphanumdash).to_string(); if (fieldname.empty()) return add_error("expected fieldname"); } @@ -205,7 +205,7 @@ namespace vcpkg::Paragraphs { paragraphs.emplace_back(); get_paragraph(paragraphs.back()); - match_zero_or_more(is_lineend); + match_while(is_lineend); } if (get_error()) return get_error()->format(); diff --git a/src/vcpkg/platform-expression.cpp b/src/vcpkg/platform-expression.cpp index 69ae0e43f2..fab65910a1 100644 --- a/src/vcpkg/platform-expression.cpp +++ b/src/vcpkg/platform-expression.cpp @@ -222,7 +222,7 @@ namespace vcpkg::PlatformExpression // { "and", optional-whitespace, platform-expression-not } // { "or", platform-expression-binary-keyword-second-operand } } // "and" is a synonym of "&", "or" is reserved (but not yet supported) as a synonym of "|" - std::string name = match_zero_or_more(is_identifier_char).to_string(); + std::string name = match_while(is_identifier_char).to_string(); Checks::check_exit(VCPKG_LINE_INFO, !name.empty()); if (name == "and") @@ -282,7 +282,7 @@ namespace vcpkg::PlatformExpression std::unique_ptr expr_identifier() { // identifier-character, { identifier-character }, - std::string name = match_zero_or_more(is_identifier_char).to_string(); + std::string name = match_while(is_identifier_char).to_string(); if (name.empty()) { @@ -312,7 +312,7 @@ namespace vcpkg::PlatformExpression } else if (cur() == 'n') { - std::string name = match_zero_or_more(is_identifier_char).to_string(); + std::string name = match_while(is_identifier_char).to_string(); // "not" if (name == "not") @@ -630,7 +630,7 @@ namespace vcpkg::PlatformExpression ExpressionParser parser(expression, multiple_binary_operators); auto res = parser.parse(); - if (auto p = parser.extract_error()) + if (auto p = std::move(parser).extract_error()) { return p->format(); } diff --git a/src/vcpkg/sourceparagraph.cpp b/src/vcpkg/sourceparagraph.cpp index 2078c63ec3..9f381430e8 100644 --- a/src/vcpkg/sourceparagraph.cpp +++ b/src/vcpkg/sourceparagraph.cpp @@ -822,7 +822,7 @@ namespace vcpkg void eat_idstring(std::string& result, Expecting& expecting) { auto loc = cur_loc(); - auto token = match_zero_or_more(is_idstring_element); + auto token = match_while(is_idstring_element); if (Strings::starts_with(token, "DocumentRef-")) { @@ -1008,7 +1008,7 @@ namespace vcpkg { auto parser = SpdxLicenseExpressionParser(sv, ""); auto result = parser.parse(); - messages = parser.extract_messages(); + messages = std::move(parser).extract_messages(); return result; } From f573df3d50b42e7cd27d559e6efa2e180041c546 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Fri, 18 Mar 2022 17:27:21 -0700 Subject: [PATCH 09/23] Finish implementing parser in terms of ParserBase. --- include/vcpkg/base/parse.h | 4 ++ src/vcpkg/base/parse.cpp | 46 +++++++++++++++ src/vcpkg/ci-baseline.cpp | 112 ++++++++++++++++++++++++++----------- 3 files changed, 130 insertions(+), 32 deletions(-) diff --git a/include/vcpkg/base/parse.h b/include/vcpkg/base/parse.h index a23c8ab2c9..06d44184cc 100644 --- a/include/vcpkg/base/parse.h +++ b/include/vcpkg/base/parse.h @@ -122,6 +122,10 @@ namespace vcpkg return {start, m_it.pointer_to_current()}; } + bool require_character(char ch); + + bool try_match_keyword(StringView keyword_content); + StringView text() const { return m_text; } Unicode::Utf8Decoder it() const { return m_it; } char32_t cur() const { return m_it == m_it.end() ? Unicode::end_of_file : *m_it; } diff --git a/src/vcpkg/base/parse.cpp b/src/vcpkg/base/parse.cpp index 2ce36cb1f1..0944590fd3 100644 --- a/src/vcpkg/base/parse.cpp +++ b/src/vcpkg/base/parse.cpp @@ -16,6 +16,12 @@ namespace (msg::value), "Example of {value} is 'x64 & windows'", " on expression: {value}"); + + DECLARE_AND_REGISTER_MESSAGE( + ExpectedCharacterHere, + (msg::expected), + "expected is a locale-invariant delimiter; for example, the ':' or '=' in 'zlib:x64-windows=skip'", + "expected {expected} here"); } namespace vcpkg @@ -139,6 +145,46 @@ namespace vcpkg skip_newline(); } + bool ParserBase::require_character(char ch) + { + if (ch == cur()) + { + next(); + return false; + } + + add_error(msg::format(msgExpectedCharacterHere, msg::expected = ch)); + return true; + } + + bool ParserBase::try_match_keyword(StringView keyword_content) { + auto first1 = m_it; + const auto last1 = first1.end(); + + auto first2 = keyword_content.begin(); + const auto last2 = keyword_content.end(); + + for (;;) + { + if (first2 == last2) + { + if (first1 == last1 || is_whitespace(*first1)) + { + m_it = first1; + m_column += static_cast(keyword_content.size()); + return true; + } + + return false; + } + + if (first1 == last1 || *first1 == *first2) + { + return false; + } + } + } + char32_t ParserBase::next() { if (m_it == m_it.end()) diff --git a/src/vcpkg/ci-baseline.cpp b/src/vcpkg/ci-baseline.cpp index 7520789aef..326291edac 100644 --- a/src/vcpkg/ci-baseline.cpp +++ b/src/vcpkg/ci-baseline.cpp @@ -7,6 +7,14 @@ using namespace vcpkg; +namespace +{ + DECLARE_AND_REGISTER_MESSAGE(ExpectedPortName, (), "", "expected a port name here"); + DECLARE_AND_REGISTER_MESSAGE(ExpectedTripletName, (), "", "expected a triplet name here"); + DECLARE_AND_REGISTER_MESSAGE(ExpectedFailOrSkip, (), "", "expected 'fail' or 'skip' here"); + DECLARE_AND_REGISTER_MESSAGE(UnknownBaselineFileContent, (), "", "unrecognizable baseline entry; expected 'port:triplet=(fail|skip)'"); +} + namespace vcpkg { TripletExclusions::TripletExclusions(const Triplet& triplet) : triplet(triplet), exclusions() { } @@ -60,49 +68,89 @@ namespace vcpkg { std::vector result; ParserBase parser(text, origin); - while (!parser.at_eof()) + for (;;) { parser.skip_whitespace(); + if (parser.at_eof()) + { + // success + return result; + } + if (parser.cur() == '#') { parser.skip_line(); continue; } - - // port-name:triplet=(fail|skip) + + // port-name:triplet = (fail|skip)\b auto port = parser.match_while(ParserBase::is_package_name_char); - (void)port; + if (port.empty()) + { + parser.add_error(msg::format(msgExpectedPortName)); + break; + } + + if (parser.require_character(':')) + { + break; + } + + auto triplet = parser.match_while(ParserBase::is_package_name_char); + if (triplet.empty()) + { + parser.add_error(msg::format(msgExpectedTripletName)); + break; + } + + parser.skip_tabs_spaces(); + if (parser.require_character('=')) + { + break; + } + + parser.skip_tabs_spaces(); + + static constexpr StringLiteral FAIL = "fail"; + static constexpr StringLiteral SKIP = "skip"; + CiBaselineState state; + if (parser.try_match_keyword(FAIL)) + { + state = CiBaselineState::Fail; + } + else if (parser.try_match_keyword(SKIP)) + { + state = CiBaselineState::Skip; + } + else + { + parser.add_error(msg::format(msgExpectedFailOrSkip)); + break; + } + + parser.skip_tabs_spaces(); + auto trailing = parser.cur(); + if (trailing == '#') + { + parser.skip_line(); + } + else if (trailing == '\r' || trailing == '\n') + { + parser.skip_newline(); + } + else + { + parser.add_error(msg::format(msgUnknownBaselineFileContent)); + break; + } + + result.emplace_back( + CiBaselineLine{port.to_string(), Triplet::from_canonical_name(triplet.to_string()), state}); } + // failure messages = std::move(parser).extract_messages(); - //for (auto& line : lines) - //{ - // if (line.empty() || line[0] == '#') continue; - // CiBaselineLine parsed_line; - - // auto colon_loc = line.find(':'); - // Checks::check_exit(VCPKG_LINE_INFO, colon_loc != std::string::npos, "Line '%s' must contain a ':'", line); - // parsed_line.port_name = Strings::trim(StringView{line.data(), line.data() + colon_loc}).to_string(); - - // auto equal_loc = line.find('=', colon_loc + 1); - // Checks::check_exit(VCPKG_LINE_INFO, equal_loc != std::string::npos, "Line '%s' must contain a '='", line); - // parsed_line.triplet = Triplet::from_canonical_name( - // Strings::trim(StringView{line.data() + colon_loc + 1, line.data() + equal_loc}).to_string()); - - // auto baseline_value = Strings::trim(StringView{line.data() + equal_loc + 1, line.data() + line.size()}); - // if (baseline_value == "fail") - // { - // parsed_line.state = CiBaselineState::Fail; - // } - // else if (baseline_value == "skip") - // { - // parsed_line.state = CiBaselineState::Skip; - // } - // else - // { - // Checks::exit_with_message(VCPKG_LINE_INFO, "Unknown value '%s'", baseline_value); - // } - //} + result.clear(); return result; } From 13010c28ed426f986fd2f875f773bb2ea9c4a8ef Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Fri, 18 Mar 2022 19:26:13 -0700 Subject: [PATCH 10/23] Tests and bugfixes! --- include/vcpkg-test/util.h | 10 ++ include/vcpkg/base/parse.h | 3 +- include/vcpkg/base/sortedvector.h | 26 ++- include/vcpkg/ci-baseline.h | 7 + locales/messages.en.json | 9 +- locales/messages.json | 11 +- src/vcpkg-test/ci-baseline.cpp | 284 ++++++++++++++++++++++++++++++ src/vcpkg/base/parse.cpp | 14 +- src/vcpkg/ci-baseline.cpp | 17 +- 9 files changed, 358 insertions(+), 23 deletions(-) create mode 100644 src/vcpkg-test/ci-baseline.cpp diff --git a/include/vcpkg-test/util.h b/include/vcpkg-test/util.h index 5b55a364ee..a1a8d6d3cb 100644 --- a/include/vcpkg-test/util.h +++ b/include/vcpkg-test/util.h @@ -5,7 +5,9 @@ #include #include #include +#include +#include #include #include @@ -55,10 +57,18 @@ namespace Catch { static const std::string convert(const vcpkg::LocalizedString& value) { return "LL\"" + value.data() + "\""; } }; + + template<> + struct StringMaker + { + static const std::string convert(const vcpkg::PackageSpec& value) { return value.to_string(); } + }; } namespace vcpkg { + inline std::ostream& operator<<(std::ostream& os, const PackageSpec& value) { return os << value.to_string(); } + inline std::ostream& operator<<(std::ostream& os, const LocalizedString& value) { return os << "LL" << std::quoted(value.data()); diff --git a/include/vcpkg/base/parse.h b/include/vcpkg/base/parse.h index 06d44184cc..bc538651a4 100644 --- a/include/vcpkg/base/parse.h +++ b/include/vcpkg/base/parse.h @@ -67,6 +67,7 @@ namespace vcpkg std::vector warnings; void exit_if_errors_or_warnings(StringView origin) const; + bool good() const { return !error && warnings.empty(); } }; struct ParserBase @@ -118,7 +119,7 @@ namespace vcpkg { ch = next(); } - + return {start, m_it.pointer_to_current()}; } diff --git a/include/vcpkg/base/sortedvector.h b/include/vcpkg/base/sortedvector.h index 70cf2c037f..e190d221fc 100644 --- a/include/vcpkg/base/sortedvector.h +++ b/include/vcpkg/base/sortedvector.h @@ -1,9 +1,10 @@ #pragma once #include +#include +#include #include #include -#include // Add more forwarding functions to the m_data std::vector as needed. namespace vcpkg @@ -25,7 +26,7 @@ namespace vcpkg { sort_uniqueify(); } - explicit SortedVector(std::vector&& data) : m_data(std::move(data)) { sort_uniqueify(); } + explicit SortedVector(std::vector&& data) : m_data(std::move(data)), m_comp() { sort_uniqueify(); } explicit SortedVector(std::vector&& data, Compare comp) : m_data(std::move(data)), m_comp(comp) { sort_uniqueify(); @@ -43,6 +44,8 @@ namespace vcpkg sort_uniqueify(); } + SortedVector(std::initializer_list elements) : m_data(elements), m_comp() { sort_uniqueify(); } + iterator begin() const { return this->m_data.cbegin(); } iterator end() const { return this->m_data.cend(); } @@ -64,7 +67,7 @@ namespace vcpkg // This could use a more efficient merge algorithm than inplace_merge with an understanding that we will // allocate the whole result if perf becomes a problem auto merge_point = m_data.insert(m_data.end(), other.begin(), other.end()); - std::inplace_merge(m_data.begin(), merge_point, m_data.end()); + std::inplace_merge(m_data.begin(), merge_point, m_data.end(), m_comp); uniqueify(); } @@ -72,12 +75,25 @@ namespace vcpkg { auto merge_point = m_data.insert( m_data.end(), std::make_move_iterator(other.begin()), std::make_move_iterator(other.end())); - std::inplace_merge(m_data.begin(), merge_point, m_data.end()); + std::inplace_merge(m_data.begin(), merge_point, m_data.end(), m_comp); uniqueify(); } + friend bool operator==(const SortedVector& lhs, const SortedVector& rhs) { return lhs.m_data == rhs.m_data; } + friend bool operator!=(const SortedVector& lhs, const SortedVector& rhs) { return lhs.m_data != rhs.m_data; } + private: - void uniqueify() { m_data.erase(std::unique(m_data.begin(), m_data.end(), m_comp), m_data.end()); } + void uniqueify() + { + Compare comp = m_comp; + m_data.erase(std::unique(m_data.begin(), + m_data.end(), + [comp](const Ty& lhs, const Ty& rhs) { + // note that we know !comp(rhs, lhs) because m_data is sorted + return !comp(lhs, rhs); + }), + m_data.end()); + } void sort_uniqueify() { if (!std::is_sorted(m_data.begin(), m_data.end(), m_comp)) diff --git a/include/vcpkg/ci-baseline.h b/include/vcpkg/ci-baseline.h index e2c3ac271c..b11d6d20ef 100644 --- a/include/vcpkg/ci-baseline.h +++ b/include/vcpkg/ci-baseline.h @@ -19,6 +19,13 @@ namespace vcpkg std::string port_name; Triplet triplet; CiBaselineState state; + + friend bool operator==(const CiBaselineLine& lhs, const CiBaselineLine& rhs) + { + return lhs.port_name == rhs.port_name && lhs.triplet == rhs.triplet && lhs.state == rhs.state; + } + + friend bool operator!=(const CiBaselineLine& lhs, const CiBaselineLine& rhs) { return !(lhs == rhs); } }; struct TripletExclusions diff --git a/locales/messages.en.json b/locales/messages.en.json index a37bfe78c5..4504963e02 100644 --- a/locales/messages.en.json +++ b/locales/messages.en.json @@ -40,9 +40,12 @@ "ErrorRequirePackagesList": "Error: `vcpkg install` requires a list of packages to install in classic mode.", "ErrorRequirePackagesToInstall": "Error: No packages were listed for installation and no manifest was found.", "ErrorVcvarsUnsupported": "Error: in triplet {triplet}: Use of Visual Studio's Developer Prompt is unsupported on non-Windows hosts.\nDefine 'VCPKG_CMAKE_SYSTEM_NAME' or 'VCPKG_CHAINLOAD_TOOLCHAIN_FILE' in the triplet file.", + "ExpectedCharacterHere": "expected '{expected}' here", + "ExpectedFailOrSkip": "expected 'fail' or 'skip' here", + "ExpectedPortName": "expected a port name here", + "ExpectedTripletName": "expected a triplet name here", "ForceSystemBinariesOnWeirdPlatforms": "Environment variable VCPKG_FORCE_SYSTEM_BINARIES must be set on arm, s390x, and ppc64le platforms.", "FormattedParseMessageExpression": " on expression: {value}", - "FormattedParseMessageLocation": "{path}:{row}:{column}: ", "GenerateMsgErrorParsingFormatArgs": "error: parsing format string for {value}:", "GenerateMsgIncorrectComment": "message {value} has an incorrect comment:", "GenerateMsgNoArgumentValue": " {{{value}}} was specified in a comment, but was not used in the message.", @@ -81,6 +84,7 @@ "ResultsLine": " {spec}: {build_result}: {elapsed}", "SeeURL": "See {url} for more information.", "SuggestNewVersionScheme": "Use the version scheme \"{new_scheme}\" instead of \"{old_scheme}\" in port \"{package_name}\".\nUse `--{option}` to disable this check.", + "UnknownBaselineFileContent": "unrecognizable baseline entry; expected 'port:triplet=(fail|skip)'", "UnsupportedSystemName": "Error: Could not map VCPKG_CMAKE_SYSTEM_NAME '{system_name}' to a vcvarsall platform. Supported system names are '', 'Windows' and 'WindowsStore'.", "UnsupportedToolchain": "Error: in triplet {triplet}: Unable to find a valid toolchain combination.\n The requested target architecture was {arch}\n The selected Visual Studio instance is at {path}\n The available toolchain combinations are {list}\n", "UsingManifestAt": "Using manifest file at {path}.", @@ -98,5 +102,6 @@ "VersionInvalidRelaxed": "`{version}` is not a valid relaxed version (semver with arbitrary numeric element count).", "VersionInvalidSemver": "`{version}` is not a valid semantic version, consult .", "VersionSpecMismatch": "error: Failed to load port because version specs did not match\n Path: {path}\n Expected: {expected_version}\n Actual: {actual_version}", - "WarningMessage": "warning: " + "WarningMessage": "warning: ", + "WarningsTreatedAsErrors": "previous warnings being interpreted as errors" } diff --git a/locales/messages.json b/locales/messages.json index 4fe5a437d4..a5d038c059 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -71,11 +71,14 @@ "ErrorRequirePackagesToInstall": "Error: No packages were listed for installation and no manifest was found.", "ErrorVcvarsUnsupported": "Error: in triplet {triplet}: Use of Visual Studio's Developer Prompt is unsupported on non-Windows hosts.\nDefine 'VCPKG_CMAKE_SYSTEM_NAME' or 'VCPKG_CHAINLOAD_TOOLCHAIN_FILE' in the triplet file.", "_ErrorVcvarsUnsupported.comment": "example of {triplet} is 'x64-windows'.\n", + "ExpectedCharacterHere": "expected '{expected}' here", + "_ExpectedCharacterHere.comment": "{expected} is a locale-invariant delimiter; for example, the ':' or '=' in 'zlib:x64-windows=skip'\n", + "ExpectedFailOrSkip": "expected 'fail' or 'skip' here", + "ExpectedPortName": "expected a port name here", + "ExpectedTripletName": "expected a triplet name here", "ForceSystemBinariesOnWeirdPlatforms": "Environment variable VCPKG_FORCE_SYSTEM_BINARIES must be set on arm, s390x, and ppc64le platforms.", "FormattedParseMessageExpression": " on expression: {value}", "_FormattedParseMessageExpression.comment": "Example of {value} is 'x64 & windows'\n", - "FormattedParseMessageLocation": "{path}:{row}:{column}: ", - "_FormattedParseMessageLocation.comment": "{Locked}", "GenerateMsgErrorParsingFormatArgs": "error: parsing format string for {value}:", "_GenerateMsgErrorParsingFormatArgs.comment": "example of {value} 'GenerateMsgNoComment'\n", "GenerateMsgIncorrectComment": "message {value} has an incorrect comment:", @@ -137,6 +140,7 @@ "_SeeURL.comment": "example of {url} is 'https://github.com/microsoft/vcpkg'.\n", "SuggestNewVersionScheme": "Use the version scheme \"{new_scheme}\" instead of \"{old_scheme}\" in port \"{package_name}\".\nUse `--{option}` to disable this check.", "_SuggestNewVersionScheme.comment": "example of {new_scheme} is 'version'.\nexample of {old_scheme} is 'version-string'.\nexample of {package_name} is 'zlib'.\nexample of {option} is 'editable'.\n", + "UnknownBaselineFileContent": "unrecognizable baseline entry; expected 'port:triplet=(fail|skip)'", "UnsupportedSystemName": "Error: Could not map VCPKG_CMAKE_SYSTEM_NAME '{system_name}' to a vcvarsall platform. Supported system names are '', 'Windows' and 'WindowsStore'.", "_UnsupportedSystemName.comment": "example of {system_name} is 'Darwin'.\n", "UnsupportedToolchain": "Error: in triplet {triplet}: Unable to find a valid toolchain combination.\n The requested target architecture was {arch}\n The selected Visual Studio instance is at {path}\n The available toolchain combinations are {list}\n", @@ -165,5 +169,6 @@ "_VersionInvalidSemver.comment": "example of {version} is '1.3.8'.\n", "VersionSpecMismatch": "error: Failed to load port because version specs did not match\n Path: {path}\n Expected: {expected_version}\n Actual: {actual_version}", "_VersionSpecMismatch.comment": "example of {path} is '/foo/bar'.\nexample of {expected_version} is '1.3.8'.\nexample of {actual_version} is '1.3.8'.\n", - "WarningMessage": "warning: " + "WarningMessage": "warning: ", + "WarningsTreatedAsErrors": "previous warnings being interpreted as errors" } diff --git a/src/vcpkg-test/ci-baseline.cpp b/src/vcpkg-test/ci-baseline.cpp new file mode 100644 index 0000000000..3fd27ac075 --- /dev/null +++ b/src/vcpkg-test/ci-baseline.cpp @@ -0,0 +1,284 @@ +#include + +#include +#include + +#include +#include + +#include + +using namespace vcpkg; + +namespace vcpkg +{ + std::ostream& operator<<(std::ostream& os, const CiBaselineLine& line) + { + os << line.port_name << ':' << line.triplet.canonical_name() << '='; + switch (line.state) + { + case CiBaselineState::Fail: os << "fail"; break; + case CiBaselineState::Skip: os << "skip"; break; + default: Checks::unreachable(VCPKG_LINE_INFO); + } + + return os; + } +} + +TEST_CASE ("Parse Empty", "[ci-baseline]") +{ + ParseMessages m; + auto actual = parse_ci_baseline("", "test", m); + CHECK(m.good()); + CHECK(actual.empty()); +} + +namespace +{ + static constexpr StringLiteral example_input = + R"(########################################################################### +## This file defines the current expected build state of ports in CI. +## +## States +## pass - (default) the port builds in the CI system. If a port is +## missing from this file then it is assumed to build. +## fail - the port does not build in the CI system. +## This is not necessarily the same as if a port is expected to build +## on a developers machine because it may fail due to the machine +## configuration. When set to fail the CI system will still attempt +## to build the port and will report a CI failure until this file is updated. +## skip - Do not build this port in the CI system. +## This is added to ports that may be flaky or conflict with other +## ports. Please comment for why a port is skipped so it can be +## removed when the issue is resolved. +## +## +## CI tested triplets: +## arm64-windows +## arm-uwp +## x64-linux +## x64-osx +## x64-uwp +## x64-windows +## x64-windows-static +## x64-windows-static-md +## x86-windows +## + +# Add new items alphabetically + +# script ports +#vcpkg-cmake:arm64-windows=fail +#vcpkg-cmake:arm-uwp=fail +#vcpkg-cmake:x64-uwp=fail +#vcpkg-cmake:x64-windows-static=fail +#vcpkg-cmake:x64-windows-static-md=fail +#vcpkg-cmake:x86-windows=fail + +#vcpkg-cmake-config:arm64-windows=fail +#vcpkg-cmake-config:arm-uwp=fail +#vcpkg-cmake-config:x64-uwp=fail +#vcpkg-cmake-config:x64-windows-static=fail +#vcpkg-cmake-config:x64-windows-static-md=fail +#vcpkg-cmake-config:x86-windows=fail + +# other ports +# Cross compiling CI machine cannot run gen_test_char to generate apr_escape_test_char.h +apr:arm64-windows=fail +# Requires ATL for ARM64 to be installed in CI +azure-storage-cpp:arm64-windows=fail + +aubio:arm-uwp=fail +aubio:x64-uwp=fail +# broken when `python` is python3, https://github.com/microsoft/vcpkg/issues/18937 +bde:x64-linux=fail +bitserializer:x64-osx=fail +blitz:x64-uwp=fail +blitz:arm64-windows=fail +blitz:arm-uwp=fail +blosc:arm64-windows=fail +blosc:arm-uwp=fail +blosc:x64-uwp=fail +bond:arm-uwp=fail +bond:x64-osx=fail +bond:x64-uwp=fail +botan:x64-uwp=fail +breakpad:arm64-windows=fail +buck-yeh-bux:x64-linux=fail +buck-yeh-bux-mariadb-client:x64-linux=fail +caf:arm-uwp=fail +caf:x64-uwp=fail +caffe2:x86-windows=fail +caffe2:arm64-windows=fail +c-ares:arm-uwp=fail +c-ares:x64-uwp=fail +casclib:arm-uwp=fail +casclib:x64-uwp=fail +catch-classic:arm64-windows = skip +catch-classic:arm-uwp = skip +catch-classic:x64-linux = skip +catch-classic:x64-osx = skip +catch-classic:x64-uwp = skip +catch-classic:x64-windows = skip +catch-classic:x64-windows-static = skip +catch-classic:x64-windows-static-md=skip +catch-classic:x86-windows = skip +bill-made-up-another-skip:x64-linux=skip)"; // note no trailing newline + + const auto x86_windows = Triplet::from_canonical_name("x86-windows"); + const auto x64_windows = Triplet::from_canonical_name("x64-windows"); + const auto x64_windows_static = Triplet::from_canonical_name("x64-windows-static"); + const auto x64_windows_static_md = Triplet::from_canonical_name("x64-windows-static-md"); + const auto x64_uwp = Triplet::from_canonical_name("x64-uwp"); + const auto arm64_windows = Triplet::from_canonical_name("arm64-windows"); + const auto arm_uwp = Triplet::from_canonical_name("arm-uwp"); + const auto x64_osx = Triplet::from_canonical_name("x64-osx"); + const auto x64_linux = Triplet::from_canonical_name("x64-linux"); + + std::vector expected_from_example_input{ + CiBaselineLine{"apr", arm64_windows, CiBaselineState::Fail}, + CiBaselineLine{"azure-storage-cpp", arm64_windows, CiBaselineState::Fail}, + CiBaselineLine{"aubio", arm_uwp, CiBaselineState::Fail}, + CiBaselineLine{"aubio", x64_uwp, CiBaselineState::Fail}, + CiBaselineLine{"bde", x64_linux, CiBaselineState::Fail}, + CiBaselineLine{"bitserializer", x64_osx, CiBaselineState::Fail}, + CiBaselineLine{"blitz", x64_uwp, CiBaselineState::Fail}, + CiBaselineLine{"blitz", arm64_windows, CiBaselineState::Fail}, + CiBaselineLine{"blitz", arm_uwp, CiBaselineState::Fail}, + CiBaselineLine{"blosc", arm64_windows, CiBaselineState::Fail}, + CiBaselineLine{"blosc", arm_uwp, CiBaselineState::Fail}, + CiBaselineLine{"blosc", x64_uwp, CiBaselineState::Fail}, + CiBaselineLine{"bond", arm_uwp, CiBaselineState::Fail}, + CiBaselineLine{"bond", x64_osx, CiBaselineState::Fail}, + CiBaselineLine{"bond", x64_uwp, CiBaselineState::Fail}, + CiBaselineLine{"botan", x64_uwp, CiBaselineState::Fail}, + CiBaselineLine{"breakpad", arm64_windows, CiBaselineState::Fail}, + CiBaselineLine{"buck-yeh-bux", x64_linux, CiBaselineState::Fail}, + CiBaselineLine{"buck-yeh-bux-mariadb-client", x64_linux, CiBaselineState::Fail}, + CiBaselineLine{"caf", arm_uwp, CiBaselineState::Fail}, + CiBaselineLine{"caf", x64_uwp, CiBaselineState::Fail}, + CiBaselineLine{"caffe2", x86_windows, CiBaselineState::Fail}, + CiBaselineLine{"caffe2", arm64_windows, CiBaselineState::Fail}, + CiBaselineLine{"c-ares", arm_uwp, CiBaselineState::Fail}, + CiBaselineLine{"c-ares", x64_uwp, CiBaselineState::Fail}, + CiBaselineLine{"casclib", arm_uwp, CiBaselineState::Fail}, + CiBaselineLine{"casclib", x64_uwp, CiBaselineState::Fail}, + CiBaselineLine{"catch-classic", arm64_windows, CiBaselineState::Skip}, + CiBaselineLine{"catch-classic", arm_uwp, CiBaselineState::Skip}, + CiBaselineLine{"catch-classic", x64_linux, CiBaselineState::Skip}, + CiBaselineLine{"catch-classic", x64_osx, CiBaselineState::Skip}, + CiBaselineLine{"catch-classic", x64_uwp, CiBaselineState::Skip}, + CiBaselineLine{"catch-classic", x64_windows, CiBaselineState::Skip}, + CiBaselineLine{"catch-classic", x64_windows_static, CiBaselineState::Skip}, + CiBaselineLine{"catch-classic", x64_windows_static_md, CiBaselineState::Skip}, + CiBaselineLine{"catch-classic", x86_windows, CiBaselineState::Skip}, + CiBaselineLine{"bill-made-up-another-skip", x64_linux, CiBaselineState::Skip}}; +} // unnamed namespace + +TEST_CASE ("Parse Real Prefix", "[ci-baseline]") +{ + ParseMessages m; + auto actual = parse_ci_baseline(example_input, "test", m); + CHECK(m.good()); + CHECK(expected_from_example_input == actual); +} + +TEST_CASE ("Parse Real Prefix With Trailing Newline", "[ci-baseline]") +{ + ParseMessages m; + std::string newlined_input(example_input.data(), example_input.size()); + newlined_input.push_back('\n'); + auto actual = parse_ci_baseline(newlined_input, "test", m); + CHECK(m.good()); + CHECK(expected_from_example_input == actual); +} + +static void check_error(const std::string& input, const std::string& expected_error) +{ + ParseMessages m; + auto actual = parse_ci_baseline(input, "test", m); + CHECK(actual.empty()); + CHECK(m.warnings.empty()); + CHECK(m.error->format() == expected_error); +} + +TEST_CASE ("Parse Errors", "[ci-baseline]") +{ + check_error("hello", R"(test:1:6: error: expected ':' here + on expression: hello + ^ +)"); + + check_error("?example:x64-windows=fail", R"(test:1:1: error: expected a port name here + on expression: ?example:x64-windows=fail + ^ +)"); + + check_error("x64-windows:", R"(test:1:13: error: expected a triplet name here + on expression: x64-windows: + ^ +)"); + + check_error("x64-windows:#", R"(test:1:13: error: expected a triplet name here + on expression: x64-windows:# + ^ +)"); + + // clang-format off + check_error(" \tx64-windows:", R"(test:1:21: error: expected a triplet name here + on expression: )" "\t" R"(x64-windows: + )" "\t" R"( ^ +)"); + // clang-format on + + check_error("example:x64-windows = pass", R"(test:1:28: error: expected 'fail' or 'skip' here + on expression: example:x64-windows = pass + ^ +)"); + + // note that there is 'fail' but doesn't end on a word boundary: + check_error("example:x64-windows = fails", R"(test:1:28: error: expected 'fail' or 'skip' here + on expression: example:x64-windows = fails + ^ +)"); + + check_error("example:x64-windows = fail extra stuff", + R"(test:1:33: error: unrecognizable baseline entry; expected 'port:triplet=(fail|skip)' + on expression: example:x64-windows = fail extra stuff + ^ +)"); + + check_error("example:x64-windows = fail # extra stuff\n" + "example:x64-uwp=skip extra stuff\n", + R"(test:2:22: error: unrecognizable baseline entry; expected 'port:triplet=(fail|skip)' + on expression: example:x64-uwp=skip extra stuff + ^ +)"); +} + +TEST_CASE ("Applies Skips and Fails", "[ci-baseline]") +{ + ExclusionsMap exclusions; + exclusions.insert(x64_uwp); // example triplet + exclusions.insert(x64_linux); // example host triplet + auto actual_expected_failures = parse_and_apply_ci_baseline(expected_from_example_input, exclusions); + const SortedVector expected_expected_failures{ + PackageSpec{"aubio", x64_uwp}, + PackageSpec{"bde", x64_linux}, + PackageSpec{"blitz", x64_uwp}, + PackageSpec{"blosc", x64_uwp}, + PackageSpec{"bond", x64_uwp}, + PackageSpec{"botan", x64_uwp}, + PackageSpec{"buck-yeh-bux", x64_linux}, + PackageSpec{"buck-yeh-bux-mariadb-client", x64_linux}, + PackageSpec{"c-ares", x64_uwp}, + PackageSpec{"caf", x64_uwp}, + PackageSpec{"casclib", x64_uwp}, + }; + + CHECK(actual_expected_failures == expected_expected_failures); + CHECK(exclusions.triplets.size() == 2); + CHECK(exclusions.triplets[0].exclusions == SortedVector{"catch-classic"}); + CHECK(exclusions.triplets[1].exclusions == SortedVector{"catch-classic", "bill-made-up-another-skip"}); +} diff --git a/src/vcpkg/base/parse.cpp b/src/vcpkg/base/parse.cpp index 0944590fd3..f197660950 100644 --- a/src/vcpkg/base/parse.cpp +++ b/src/vcpkg/base/parse.cpp @@ -20,8 +20,8 @@ namespace DECLARE_AND_REGISTER_MESSAGE( ExpectedCharacterHere, (msg::expected), - "expected is a locale-invariant delimiter; for example, the ':' or '=' in 'zlib:x64-windows=skip'", - "expected {expected} here"); + "{expected} is a locale-invariant delimiter; for example, the ':' or '=' in 'zlib:x64-windows=skip'", + "expected '{expected}' here"); } namespace vcpkg @@ -147,7 +147,7 @@ namespace vcpkg bool ParserBase::require_character(char ch) { - if (ch == cur()) + if (static_cast(ch) == cur()) { next(); return false; @@ -157,7 +157,8 @@ namespace vcpkg return true; } - bool ParserBase::try_match_keyword(StringView keyword_content) { + bool ParserBase::try_match_keyword(StringView keyword_content) + { auto first1 = m_it; const auto last1 = first1.end(); @@ -178,10 +179,13 @@ namespace vcpkg return false; } - if (first1 == last1 || *first1 == *first2) + if (first1 == last1 || *first1 != static_cast(*first2)) { return false; } + + ++first1; + ++first2; } } diff --git a/src/vcpkg/ci-baseline.cpp b/src/vcpkg/ci-baseline.cpp index 326291edac..20ff40f45f 100644 --- a/src/vcpkg/ci-baseline.cpp +++ b/src/vcpkg/ci-baseline.cpp @@ -12,7 +12,10 @@ namespace DECLARE_AND_REGISTER_MESSAGE(ExpectedPortName, (), "", "expected a port name here"); DECLARE_AND_REGISTER_MESSAGE(ExpectedTripletName, (), "", "expected a triplet name here"); DECLARE_AND_REGISTER_MESSAGE(ExpectedFailOrSkip, (), "", "expected 'fail' or 'skip' here"); - DECLARE_AND_REGISTER_MESSAGE(UnknownBaselineFileContent, (), "", "unrecognizable baseline entry; expected 'port:triplet=(fail|skip)'"); + DECLARE_AND_REGISTER_MESSAGE(UnknownBaselineFileContent, + (), + "", + "unrecognizable baseline entry; expected 'port:triplet=(fail|skip)'"); } namespace vcpkg @@ -138,7 +141,7 @@ namespace vcpkg { parser.skip_newline(); } - else + else if (trailing != Unicode::end_of_file) { parser.add_error(msg::format(msgUnknownBaselineFileContent)); break; @@ -157,17 +160,17 @@ namespace vcpkg SortedVector parse_and_apply_ci_baseline(View lines, ExclusionsMap& exclusions_map) { std::vector expected_failures; - std::map> added_known_fails; + std::map> added_exclusions; for (const auto& triplet_entry : exclusions_map.triplets) { - added_known_fails.emplace( + added_exclusions.emplace( std::piecewise_construct, std::forward_as_tuple(triplet_entry.triplet), std::tuple<>{}); } for (auto& line : lines) { - auto triplet_match = added_known_fails.find(line.triplet); - if (triplet_match != added_known_fails.end()) + auto triplet_match = added_exclusions.find(line.triplet); + if (triplet_match != added_exclusions.end()) { if (line.state == CiBaselineState::Skip) { @@ -183,7 +186,7 @@ namespace vcpkg for (auto& triplet_entry : exclusions_map.triplets) { triplet_entry.exclusions.append( - SortedVector(std::move(added_known_fails.find(triplet_entry.triplet)->second))); + SortedVector(std::move(added_exclusions.find(triplet_entry.triplet)->second))); } return SortedVector{std::move(expected_failures)}; From 2bc6f621c8bcfd43e87cd9b6418639901edf061f Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Mon, 21 Mar 2022 09:32:18 -0700 Subject: [PATCH 11/23] Fix MacOS and Linux build failures. --- src/vcpkg-test/ci-baseline.cpp | 227 +++++++++++++++++---------------- 1 file changed, 114 insertions(+), 113 deletions(-) diff --git a/src/vcpkg-test/ci-baseline.cpp b/src/vcpkg-test/ci-baseline.cpp index 3fd27ac075..a7550ae943 100644 --- a/src/vcpkg-test/ci-baseline.cpp +++ b/src/vcpkg-test/ci-baseline.cpp @@ -6,6 +6,7 @@ #include #include +#include #include using namespace vcpkg; @@ -37,92 +38,92 @@ TEST_CASE ("Parse Empty", "[ci-baseline]") namespace { static constexpr StringLiteral example_input = - R"(########################################################################### -## This file defines the current expected build state of ports in CI. -## -## States -## pass - (default) the port builds in the CI system. If a port is -## missing from this file then it is assumed to build. -## fail - the port does not build in the CI system. -## This is not necessarily the same as if a port is expected to build -## on a developers machine because it may fail due to the machine -## configuration. When set to fail the CI system will still attempt -## to build the port and will report a CI failure until this file is updated. -## skip - Do not build this port in the CI system. -## This is added to ports that may be flaky or conflict with other -## ports. Please comment for why a port is skipped so it can be -## removed when the issue is resolved. -## -## -## CI tested triplets: -## arm64-windows -## arm-uwp -## x64-linux -## x64-osx -## x64-uwp -## x64-windows -## x64-windows-static -## x64-windows-static-md -## x86-windows -## - -# Add new items alphabetically - -# script ports -#vcpkg-cmake:arm64-windows=fail -#vcpkg-cmake:arm-uwp=fail -#vcpkg-cmake:x64-uwp=fail -#vcpkg-cmake:x64-windows-static=fail -#vcpkg-cmake:x64-windows-static-md=fail -#vcpkg-cmake:x86-windows=fail - -#vcpkg-cmake-config:arm64-windows=fail -#vcpkg-cmake-config:arm-uwp=fail -#vcpkg-cmake-config:x64-uwp=fail -#vcpkg-cmake-config:x64-windows-static=fail -#vcpkg-cmake-config:x64-windows-static-md=fail -#vcpkg-cmake-config:x86-windows=fail - -# other ports -# Cross compiling CI machine cannot run gen_test_char to generate apr_escape_test_char.h -apr:arm64-windows=fail -# Requires ATL for ARM64 to be installed in CI -azure-storage-cpp:arm64-windows=fail - -aubio:arm-uwp=fail -aubio:x64-uwp=fail -# broken when `python` is python3, https://github.com/microsoft/vcpkg/issues/18937 -bde:x64-linux=fail -bitserializer:x64-osx=fail -blitz:x64-uwp=fail -blitz:arm64-windows=fail -blitz:arm-uwp=fail -blosc:arm64-windows=fail -blosc:arm-uwp=fail -blosc:x64-uwp=fail -bond:arm-uwp=fail -bond:x64-osx=fail -bond:x64-uwp=fail -botan:x64-uwp=fail -breakpad:arm64-windows=fail -buck-yeh-bux:x64-linux=fail -buck-yeh-bux-mariadb-client:x64-linux=fail -caf:arm-uwp=fail -caf:x64-uwp=fail -caffe2:x86-windows=fail -caffe2:arm64-windows=fail -c-ares:arm-uwp=fail -c-ares:x64-uwp=fail -casclib:arm-uwp=fail -casclib:x64-uwp=fail -catch-classic:arm64-windows = skip -catch-classic:arm-uwp = skip -catch-classic:x64-linux = skip -catch-classic:x64-osx = skip -catch-classic:x64-uwp = skip -catch-classic:x64-windows = skip -catch-classic:x64-windows-static = skip -catch-classic:x64-windows-static-md=skip + R"(########################################################################### +## This file defines the current expected build state of ports in CI. +## +## States +## pass - (default) the port builds in the CI system. If a port is +## missing from this file then it is assumed to build. +## fail - the port does not build in the CI system. +## This is not necessarily the same as if a port is expected to build +## on a developers machine because it may fail due to the machine +## configuration. When set to fail the CI system will still attempt +## to build the port and will report a CI failure until this file is updated. +## skip - Do not build this port in the CI system. +## This is added to ports that may be flaky or conflict with other +## ports. Please comment for why a port is skipped so it can be +## removed when the issue is resolved. +## +## +## CI tested triplets: +## arm64-windows +## arm-uwp +## x64-linux +## x64-osx +## x64-uwp +## x64-windows +## x64-windows-static +## x64-windows-static-md +## x86-windows +## + +# Add new items alphabetically + +# script ports +#vcpkg-cmake:arm64-windows=fail +#vcpkg-cmake:arm-uwp=fail +#vcpkg-cmake:x64-uwp=fail +#vcpkg-cmake:x64-windows-static=fail +#vcpkg-cmake:x64-windows-static-md=fail +#vcpkg-cmake:x86-windows=fail + +#vcpkg-cmake-config:arm64-windows=fail +#vcpkg-cmake-config:arm-uwp=fail +#vcpkg-cmake-config:x64-uwp=fail +#vcpkg-cmake-config:x64-windows-static=fail +#vcpkg-cmake-config:x64-windows-static-md=fail +#vcpkg-cmake-config:x86-windows=fail + +# other ports +# Cross compiling CI machine cannot run gen_test_char to generate apr_escape_test_char.h +apr:arm64-windows=fail +# Requires ATL for ARM64 to be installed in CI +azure-storage-cpp:arm64-windows=fail + +aubio:arm-uwp=fail +aubio:x64-uwp=fail +# broken when `python` is python3, https://github.com/microsoft/vcpkg/issues/18937 +bde:x64-linux=fail +bitserializer:x64-osx=fail +blitz:x64-uwp=fail +blitz:arm64-windows=fail +blitz:arm-uwp=fail +blosc:arm64-windows=fail +blosc:arm-uwp=fail +blosc:x64-uwp=fail +bond:arm-uwp=fail +bond:x64-osx=fail +bond:x64-uwp=fail +botan:x64-uwp=fail +breakpad:arm64-windows=fail +buck-yeh-bux:x64-linux=fail +buck-yeh-bux-mariadb-client:x64-linux=fail +caf:arm-uwp=fail +caf:x64-uwp=fail +caffe2:x86-windows=fail +caffe2:arm64-windows=fail +c-ares:arm-uwp=fail +c-ares:x64-uwp=fail +casclib:arm-uwp=fail +casclib:x64-uwp=fail +catch-classic:arm64-windows = skip +catch-classic:arm-uwp = skip +catch-classic:x64-linux = skip +catch-classic:x64-osx = skip +catch-classic:x64-uwp = skip +catch-classic:x64-windows = skip +catch-classic:x64-windows-static = skip +catch-classic:x64-windows-static-md=skip catch-classic:x86-windows = skip bill-made-up-another-skip:x64-linux=skip)"; // note no trailing newline @@ -205,55 +206,55 @@ static void check_error(const std::string& input, const std::string& expected_er TEST_CASE ("Parse Errors", "[ci-baseline]") { - check_error("hello", R"(test:1:6: error: expected ':' here - on expression: hello - ^ + check_error("hello", R"(test:1:6: error: expected ':' here + on expression: hello + ^ )"); - check_error("?example:x64-windows=fail", R"(test:1:1: error: expected a port name here - on expression: ?example:x64-windows=fail - ^ + check_error("?example:x64-windows=fail", R"(test:1:1: error: expected a port name here + on expression: ?example:x64-windows=fail + ^ )"); - check_error("x64-windows:", R"(test:1:13: error: expected a triplet name here - on expression: x64-windows: - ^ + check_error("x64-windows:", R"(test:1:13: error: expected a triplet name here + on expression: x64-windows: + ^ )"); - check_error("x64-windows:#", R"(test:1:13: error: expected a triplet name here - on expression: x64-windows:# - ^ + check_error("x64-windows:#", R"(test:1:13: error: expected a triplet name here + on expression: x64-windows:# + ^ )"); // clang-format off - check_error(" \tx64-windows:", R"(test:1:21: error: expected a triplet name here - on expression: )" "\t" R"(x64-windows: - )" "\t" R"( ^ + check_error(" \tx64-windows:", R"(test:1:21: error: expected a triplet name here + on expression: )" "\t" R"(x64-windows: + )" "\t" R"( ^ )"); // clang-format on - check_error("example:x64-windows = pass", R"(test:1:28: error: expected 'fail' or 'skip' here - on expression: example:x64-windows = pass - ^ + check_error("example:x64-windows = pass", R"(test:1:28: error: expected 'fail' or 'skip' here + on expression: example:x64-windows = pass + ^ )"); // note that there is 'fail' but doesn't end on a word boundary: - check_error("example:x64-windows = fails", R"(test:1:28: error: expected 'fail' or 'skip' here - on expression: example:x64-windows = fails - ^ + check_error("example:x64-windows = fails", R"(test:1:28: error: expected 'fail' or 'skip' here + on expression: example:x64-windows = fails + ^ )"); check_error("example:x64-windows = fail extra stuff", - R"(test:1:33: error: unrecognizable baseline entry; expected 'port:triplet=(fail|skip)' - on expression: example:x64-windows = fail extra stuff - ^ + R"(test:1:33: error: unrecognizable baseline entry; expected 'port:triplet=(fail|skip)' + on expression: example:x64-windows = fail extra stuff + ^ )"); check_error("example:x64-windows = fail # extra stuff\n" "example:x64-uwp=skip extra stuff\n", - R"(test:2:22: error: unrecognizable baseline entry; expected 'port:triplet=(fail|skip)' - on expression: example:x64-uwp=skip extra stuff - ^ + R"(test:2:22: error: unrecognizable baseline entry; expected 'port:triplet=(fail|skip)' + on expression: example:x64-uwp=skip extra stuff + ^ )"); } From f3dea2a55b9991aaea52fc474e9bfca34afc70f4 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Mon, 21 Mar 2022 09:47:11 -0700 Subject: [PATCH 12/23] Remove std::cerr. --- src/vcpkg/commands.ci.cpp | 88 ++++++++++++++++++++++++++------------- 1 file changed, 60 insertions(+), 28 deletions(-) diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 65c2b624aa..9ba030d6ac 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -28,7 +28,7 @@ #include #include -#include +#include using namespace vcpkg; @@ -79,6 +79,23 @@ namespace private: Path base_path; }; + + DECLARE_AND_REGISTER_MESSAGE( + CiBaselineRegressionHeader, + (), + "Printed before a series of CiBaselineRegression and/or CiBaselineUnexpectedPass messages.", + "REGRESSIONS:"); + + DECLARE_AND_REGISTER_MESSAGE( + CiBaselineRegression, + (msg::spec, msg::build_result, msg::path), + "", + "REGRESSION: {spec} failed with {build_result}. If expected, add {spec}=fail to {path}."); + + DECLARE_AND_REGISTER_MESSAGE(CiBaselineUnexpectedPass, + (msg::spec, msg::path), + "", + "PASSING, REMOVE FROM FAIL LIST: {spec} ({path})."); } namespace vcpkg::Commands::CI @@ -460,6 +477,47 @@ namespace vcpkg::Commands::CI return result; } + static void print_baseline_regressions(const TripletAndSummary& result, + const SortedVector& expected_failures, + const std::string& ci_baseline_file_name, + bool allow_unexpected_passing) + { + LocalizedString output = msg::format(msgCiBaselineRegressionHeader); + output.appendnl(); + for (auto&& port_result : result.summary.results) + { + switch (port_result.build_result.code) + { + case Build::BuildResult::BUILD_FAILED: + case Build::BuildResult::POST_BUILD_CHECKS_FAILED: + case Build::BuildResult::FILE_CONFLICTS: + if (!expected_failures.contains(port_result.spec)) + { + output.append(msg::format( + msgCiBaselineRegression, + msg::spec = port_result.spec.to_string(), + msg::build_result = + Build::to_string_locale_invariant(port_result.build_result.code).to_string(), + msg::path = ci_baseline_file_name)); + output.appendnl(); + } + break; + case Build::BuildResult::SUCCEEDED: + if (!allow_unexpected_passing && expected_failures.contains(port_result.spec)) + { + output.append(msg::format(msgCiBaselineUnexpectedPass, + msg::spec = port_result.spec.to_string(), + msg::path = ci_baseline_file_name)); + output.appendnl(); + } + break; + default: break; + } + } + + fputs(output.data().c_str(), stderr); + } + void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths, Triplet target_triplet, @@ -688,33 +746,7 @@ namespace vcpkg::Commands::CI if (baseline_iter != settings.end()) { - print2("\nREGRESSIONS:\n"); - for (auto&& port_result : result.summary.results) - { - switch (port_result.build_result.code) - { - case Build::BuildResult::BUILD_FAILED: - case Build::BuildResult::POST_BUILD_CHECKS_FAILED: - case Build::BuildResult::FILE_CONFLICTS: - if (!expected_failures.contains(port_result.spec)) - { - std::cerr - << " REGRESSION: " << port_result.spec.to_string() << " failed with " - << Build::to_string_locale_invariant(port_result.build_result.code).to_string() - << ". If expected, add " << port_result.spec.to_string() << "=fail to " - << baseline_iter->second << std::endl; - } - break; - case Build::BuildResult::SUCCEEDED: - if (!allow_unexpected_passing && expected_failures.contains(port_result.spec)) - { - std::cerr << " PASSING, REMOVE FROM FAIL LIST: " << port_result.spec.to_string() - << " (" << baseline_iter->second << ")" << std::endl; - } - break; - default: break; - } - } + print_baseline_regressions(result, expected_failures, baseline_iter->second, allow_unexpected_passing); } } From 1f41c1149172579819e89d21b681d616ed626a69 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Mon, 21 Mar 2022 09:49:16 -0700 Subject: [PATCH 13/23] Generate messages map --- locales/messages.en.json | 3 +++ locales/messages.json | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/locales/messages.en.json b/locales/messages.en.json index 4504963e02..d8f5563283 100644 --- a/locales/messages.en.json +++ b/locales/messages.en.json @@ -25,6 +25,9 @@ "ChecksLineInfo": "{vcpkg_line_info}: ", "ChecksUnreachableCode": "unreachable code was reached", "ChecksUpdateVcpkg": "updating vcpkg by rerunning bootstrap-vcpkg may resolve this failure.", + "CiBaselineRegression": "REGRESSION: {spec} failed with {build_result}. If expected, add {spec}=fail to {path}.", + "CiBaselineRegressionHeader": "REGRESSIONS:", + "CiBaselineUnexpectedPass": "PASSING, REMOVE FROM FAIL LIST: {spec} ({path}).", "CouldNotDeduceNugetIdAndVersion": "Could not deduce nuget id and version from filename: {path}", "EmptyLicenseExpression": "SPDX license expression was empty.", "ErrorIndividualPackagesUnsupported": "Error: In manifest mode, `vcpkg install` does not support individual package arguments.\nTo install additional packages, edit vcpkg.json and then run `vcpkg install` without any package arguments.", diff --git a/locales/messages.json b/locales/messages.json index a5d038c059..867d835364 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -47,6 +47,12 @@ "_ChecksLineInfo.comment": "{Locked}", "ChecksUnreachableCode": "unreachable code was reached", "ChecksUpdateVcpkg": "updating vcpkg by rerunning bootstrap-vcpkg may resolve this failure.", + "CiBaselineRegression": "REGRESSION: {spec} failed with {build_result}. If expected, add {spec}=fail to {path}.", + "_CiBaselineRegression.comment": "example of {spec} is 'zlib:x64-windows'.\nexample of {build_result} is 'One of the BuildResultXxx messages (such as BuildResultSucceeded/SUCCEEDED)'.\nexample of {path} is '/foo/bar'.\n", + "CiBaselineRegressionHeader": "REGRESSIONS:", + "_CiBaselineRegressionHeader.comment": "Printed before a series of CiBaselineRegression and/or CiBaselineUnexpectedPass messages.\n", + "CiBaselineUnexpectedPass": "PASSING, REMOVE FROM FAIL LIST: {spec} ({path}).", + "_CiBaselineUnexpectedPass.comment": "example of {spec} is 'zlib:x64-windows'.\nexample of {path} is '/foo/bar'.\n", "CouldNotDeduceNugetIdAndVersion": "Could not deduce nuget id and version from filename: {path}", "_CouldNotDeduceNugetIdAndVersion.comment": "example of {path} is '/foo/bar'.\n", "EmptyLicenseExpression": "SPDX license expression was empty.", From c2078280f0e758131810cca6a4f092a8d30d96ef Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Mon, 21 Mar 2022 09:54:32 -0700 Subject: [PATCH 14/23] Try to fix macos and linux again. --- src/vcpkg-test/ci-baseline.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vcpkg-test/ci-baseline.cpp b/src/vcpkg-test/ci-baseline.cpp index a7550ae943..335494abce 100644 --- a/src/vcpkg-test/ci-baseline.cpp +++ b/src/vcpkg-test/ci-baseline.cpp @@ -13,7 +13,7 @@ using namespace vcpkg; namespace vcpkg { - std::ostream& operator<<(std::ostream& os, const CiBaselineLine& line) + static std::ostream& operator<<(std::ostream& os, const CiBaselineLine& line) { os << line.port_name << ':' << line.triplet.canonical_name() << '='; switch (line.state) From b089a02cf585fdbd28fe6b7f89049e82a6b45587 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Mon, 21 Mar 2022 10:30:13 -0700 Subject: [PATCH 15/23] Ensure all named triplets are in the exclusions map. --- src/vcpkg/commands.ci.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 9ba030d6ac..be99c3c965 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -454,10 +454,10 @@ namespace vcpkg::Commands::CI ExclusionsMap& exclusions_map) { auto it_exclusions = settings.find(opt); - if (it_exclusions != settings.end()) - { - exclusions_map.insert(triplet, SortedVector(Strings::split(it_exclusions->second, ','))); - } + exclusions_map.insert(triplet, + it_exclusions == settings.end() + ? SortedVector{} + : SortedVector(Strings::split(it_exclusions->second, ','))); } static Optional parse_skipped_cascade_count(const std::unordered_map& settings) From bfbbe6ecf875a6b28e915621dde54a8f41c691d2 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Tue, 22 Mar 2022 11:25:45 -0700 Subject: [PATCH 16/23] Fix bad merge. --- include/vcpkg/base/parse.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/vcpkg/base/parse.h b/include/vcpkg/base/parse.h index 4c61a03e3c..9425ea5e49 100644 --- a/include/vcpkg/base/parse.h +++ b/include/vcpkg/base/parse.h @@ -44,6 +44,8 @@ namespace vcpkg Unicode::Utf8Decoder start_of_line; int row; int column; + }; + struct ParseMessage { SourceLoc location = {}; From b99de1d2850661494f1f5353c5dcfe838b8aab99 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Tue, 22 Mar 2022 17:53:56 -0700 Subject: [PATCH 17/23] Require the caller to pass the predicate in sorted vector rather than storing it. --- include/vcpkg/base/sortedvector.h | 66 ++++++++++++++++++------------- src/vcpkg-test/update.cpp | 16 ++++---- src/vcpkg/update.cpp | 4 +- src/vcpkg/visualstudio.cpp | 4 +- 4 files changed, 51 insertions(+), 39 deletions(-) diff --git a/include/vcpkg/base/sortedvector.h b/include/vcpkg/base/sortedvector.h index e190d221fc..7fd0828241 100644 --- a/include/vcpkg/base/sortedvector.h +++ b/include/vcpkg/base/sortedvector.h @@ -9,42 +9,47 @@ // Add more forwarding functions to the m_data std::vector as needed. namespace vcpkg { - template> + template struct SortedVector { using size_type = typename std::vector::size_type; using iterator = typename std::vector::const_iterator; - SortedVector() : m_data(), m_comp() { } + SortedVector() = default; SortedVector(const SortedVector&) = default; SortedVector(SortedVector&&) = default; SortedVector& operator=(const SortedVector&) = default; SortedVector& operator=(SortedVector&&) = default; - explicit SortedVector(const std::vector& data) : m_data(data), m_comp() { sort_uniqueify(); } - explicit SortedVector(const std::vector& data, Compare comp) : m_data(data), m_comp(comp) + explicit SortedVector(const std::vector& data) : m_data(data) { sort_uniqueify(std::less<>{}); } + + template + SortedVector(const std::vector& data, Compare comp) : m_data(data) { - sort_uniqueify(); + sort_uniqueify(comp); } - explicit SortedVector(std::vector&& data) : m_data(std::move(data)), m_comp() { sort_uniqueify(); } - explicit SortedVector(std::vector&& data, Compare comp) : m_data(std::move(data)), m_comp(comp) + + explicit SortedVector(std::vector&& data) : m_data(std::move(data)) { sort_uniqueify(std::less<>{}); } + + template + explicit SortedVector(std::vector&& data, Compare comp) : m_data(std::move(data)) { - sort_uniqueify(); + sort_uniqueify(comp); } template - explicit SortedVector(InIt first, InIt last) : m_data(first, last), m_comp() + explicit SortedVector(InIt first, InIt last) : m_data(first, last) { - sort_uniqueify(); + sort_uniqueify(std::less<>{}); } - template - explicit SortedVector(InIt first, InIt last, Compare comp) : m_data(first, last), m_comp(comp) + template + explicit SortedVector(InIt first, InIt last, Compare comp) : m_data(first, last) { - sort_uniqueify(); + sort_uniqueify(comp); } - SortedVector(std::initializer_list elements) : m_data(elements), m_comp() { sort_uniqueify(); } + SortedVector(std::initializer_list elements) : m_data(elements) { sort_uniqueify(std::less<>{}); } iterator begin() const { return this->m_data.cbegin(); } @@ -62,30 +67,36 @@ namespace vcpkg bool contains(const Ty& element) const { return std::binary_search(m_data.begin(), m_data.end(), element); } - void append(const SortedVector& other) + void append(const SortedVector& other) { append(other, std::less<>{}); } + + template + void append(const SortedVector& other, Compare comp) { // This could use a more efficient merge algorithm than inplace_merge with an understanding that we will // allocate the whole result if perf becomes a problem auto merge_point = m_data.insert(m_data.end(), other.begin(), other.end()); - std::inplace_merge(m_data.begin(), merge_point, m_data.end(), m_comp); - uniqueify(); + std::inplace_merge(m_data.begin(), merge_point, m_data.end(), comp); + uniqueify(comp); } - void append(SortedVector&& other) + void append(SortedVector&& other) { append(std::move(other), std::less<>{}); } + + template + void append(SortedVector&& other, Compare comp) { auto merge_point = m_data.insert( m_data.end(), std::make_move_iterator(other.begin()), std::make_move_iterator(other.end())); - std::inplace_merge(m_data.begin(), merge_point, m_data.end(), m_comp); - uniqueify(); + std::inplace_merge(m_data.begin(), merge_point, m_data.end(), comp); + uniqueify(comp); } friend bool operator==(const SortedVector& lhs, const SortedVector& rhs) { return lhs.m_data == rhs.m_data; } friend bool operator!=(const SortedVector& lhs, const SortedVector& rhs) { return lhs.m_data != rhs.m_data; } private: - void uniqueify() + template + void uniqueify(Compare comp) { - Compare comp = m_comp; m_data.erase(std::unique(m_data.begin(), m_data.end(), [comp](const Ty& lhs, const Ty& rhs) { @@ -94,17 +105,18 @@ namespace vcpkg }), m_data.end()); } - void sort_uniqueify() + + template + void sort_uniqueify(Compare comp) { - if (!std::is_sorted(m_data.begin(), m_data.end(), m_comp)) + if (!std::is_sorted(m_data.begin(), m_data.end(), comp)) { - std::sort(m_data.begin(), m_data.end(), m_comp); + std::sort(m_data.begin(), m_data.end(), comp); } - uniqueify(); + uniqueify(comp); } std::vector m_data; - Compare m_comp; }; } diff --git a/src/vcpkg-test/update.cpp b/src/vcpkg-test/update.cpp index c3561b6bb5..5db1125994 100644 --- a/src/vcpkg-test/update.cpp +++ b/src/vcpkg-test/update.cpp @@ -27,8 +27,8 @@ TEST_CASE ("find outdated packages basic", "[update]") map.emplace("a", SourceControlFileAndLocation{std::move(scf), ""}); PortFileProvider::MapPortFileProvider provider(map); - auto pkgs = SortedVector( - Update::find_outdated_packages(provider, status_db), &OutdatedPackage::compare_by_name); + auto pkgs = SortedVector(Update::find_outdated_packages(provider, status_db), + &OutdatedPackage::compare_by_name); REQUIRE(pkgs.size() == 1); REQUIRE(pkgs[0].version_diff.left.to_string() == "2"); @@ -51,8 +51,8 @@ TEST_CASE ("find outdated packages features", "[update]") map.emplace("a", SourceControlFileAndLocation{std::move(scf), ""}); PortFileProvider::MapPortFileProvider provider(map); - auto pkgs = SortedVector( - Update::find_outdated_packages(provider, status_db), &OutdatedPackage::compare_by_name); + auto pkgs = SortedVector(Update::find_outdated_packages(provider, status_db), + &OutdatedPackage::compare_by_name); REQUIRE(pkgs.size() == 1); REQUIRE(pkgs[0].version_diff.left.to_string() == "2"); @@ -77,8 +77,8 @@ TEST_CASE ("find outdated packages features 2", "[update]") map.emplace("a", SourceControlFileAndLocation{std::move(scf), ""}); PortFileProvider::MapPortFileProvider provider(map); - auto pkgs = SortedVector( - Update::find_outdated_packages(provider, status_db), &OutdatedPackage::compare_by_name); + auto pkgs = SortedVector(Update::find_outdated_packages(provider, status_db), + &OutdatedPackage::compare_by_name); REQUIRE(pkgs.size() == 1); REQUIRE(pkgs[0].version_diff.left.to_string() == "2"); @@ -98,8 +98,8 @@ TEST_CASE ("find outdated packages none", "[update]") map.emplace("a", SourceControlFileAndLocation{std::move(scf), ""}); PortFileProvider::MapPortFileProvider provider(map); - auto pkgs = SortedVector( - Update::find_outdated_packages(provider, status_db), &OutdatedPackage::compare_by_name); + auto pkgs = SortedVector(Update::find_outdated_packages(provider, status_db), + &OutdatedPackage::compare_by_name); REQUIRE(pkgs.size() == 0); } diff --git a/src/vcpkg/update.cpp b/src/vcpkg/update.cpp index 180abc7e9b..0b2c179645 100644 --- a/src/vcpkg/update.cpp +++ b/src/vcpkg/update.cpp @@ -70,8 +70,8 @@ namespace vcpkg::Update PortFileProvider::PathsPortFileProvider provider(paths, args.overlay_ports); - const auto outdated_packages = SortedVector( - find_outdated_packages(provider, status_db), &OutdatedPackage::compare_by_name); + const auto outdated_packages = SortedVector(find_outdated_packages(provider, status_db), + &OutdatedPackage::compare_by_name); if (outdated_packages.empty()) { diff --git a/src/vcpkg/visualstudio.cpp b/src/vcpkg/visualstudio.cpp index c9e8a70cc7..fb6da70fd8 100644 --- a/src/vcpkg/visualstudio.cpp +++ b/src/vcpkg/visualstudio.cpp @@ -210,8 +210,8 @@ namespace vcpkg::VisualStudio std::vector& paths_examined = ret.paths_examined; std::vector& found_toolsets = ret.toolsets; - const SortedVector sorted{ - get_visual_studio_instances_internal(fs), VisualStudioInstance::preferred_first_comparator}; + const SortedVector sorted{get_visual_studio_instances_internal(fs), + VisualStudioInstance::preferred_first_comparator}; const bool v140_is_available = Util::find_if(sorted, [&](const VisualStudioInstance& vs_instance) { return vs_instance.major_version() == "14"; From e97eac1029fc3a1f152361292b81c56cb631702b Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Tue, 22 Mar 2022 17:54:12 -0700 Subject: [PATCH 18/23] Revert the match_until change. --- include/vcpkg/base/parse.h | 7 ++++++- src/vcpkg/base/parse.cpp | 2 +- src/vcpkg/binarycaching.cpp | 2 +- src/vcpkg/paragraphs.cpp | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/include/vcpkg/base/parse.h b/include/vcpkg/base/parse.h index 9425ea5e49..7f4d18bc2e 100644 --- a/include/vcpkg/base/parse.h +++ b/include/vcpkg/base/parse.h @@ -73,7 +73,6 @@ namespace vcpkg static constexpr bool is_icase_alpha(char32_t ch) { return is_lower_alpha(ch) || is_upper_alpha(ch); } static constexpr bool is_ascii_digit(char32_t ch) { return ch >= '0' && ch <= '9'; } static constexpr bool is_lineend(char32_t ch) { return ch == '\r' || ch == '\n' || ch == Unicode::end_of_file; } - static constexpr bool is_not_lineend(char32_t ch) { return !is_lineend(ch); } static constexpr bool is_alphanum(char32_t ch) { return is_icase_alpha(ch) || is_ascii_digit(ch); } static constexpr bool is_alphadash(char32_t ch) { return is_icase_alpha(ch) || ch == '-'; } static constexpr bool is_alphanumdash(char32_t ch) { return is_alphanum(ch) || ch == '-'; } @@ -107,6 +106,12 @@ namespace vcpkg return {start, m_it.pointer_to_current()}; } + template + StringView match_until(Pred p) + { + return match_while([p](char32_t ch) { return !p(ch); }); + } + bool require_character(char ch); bool try_match_keyword(StringView keyword_content); diff --git a/src/vcpkg/base/parse.cpp b/src/vcpkg/base/parse.cpp index f197660950..24b43f358b 100644 --- a/src/vcpkg/base/parse.cpp +++ b/src/vcpkg/base/parse.cpp @@ -141,7 +141,7 @@ namespace vcpkg } void ParserBase::skip_line() { - match_while(is_not_lineend); + match_until(is_lineend); skip_newline(); } diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 4feaf94684..e2684d5f30 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -90,7 +90,7 @@ namespace std::string segment; for (;;) { - auto n = match_while([](char32_t ch) { return ch != ',' && ch != '`' && ch != ';'; }); + auto n = match_until([](char32_t ch) { return ch == ',' || ch == '`' || ch == ';'; }); Strings::append(segment, n); auto ch = cur(); if (ch == Unicode::end_of_file || ch == ',' || ch == ';') diff --git a/src/vcpkg/paragraphs.cpp b/src/vcpkg/paragraphs.cpp index 7ee7781aea..8378de5eb8 100644 --- a/src/vcpkg/paragraphs.cpp +++ b/src/vcpkg/paragraphs.cpp @@ -151,7 +151,7 @@ namespace vcpkg::Paragraphs do { // scan to end of current line (it is part of the field value) - Strings::append(fieldvalue, match_while(is_not_lineend)); + Strings::append(fieldvalue, match_until(is_lineend)); skip_newline(); if (cur() != ' ') return; From ffa2c8b40cc8a37f47866098e7caba78ad2379bb Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 23 Mar 2022 12:56:43 -0700 Subject: [PATCH 19/23] Revert "Require the caller to pass the predicate in sorted vector rather than storing it." This reverts commit b99de1d2850661494f1f5353c5dcfe838b8aab99. --- include/vcpkg/base/sortedvector.h | 66 +++++++++++++------------------ src/vcpkg-test/update.cpp | 16 ++++---- src/vcpkg/update.cpp | 4 +- src/vcpkg/visualstudio.cpp | 4 +- 4 files changed, 39 insertions(+), 51 deletions(-) diff --git a/include/vcpkg/base/sortedvector.h b/include/vcpkg/base/sortedvector.h index 7fd0828241..e190d221fc 100644 --- a/include/vcpkg/base/sortedvector.h +++ b/include/vcpkg/base/sortedvector.h @@ -9,47 +9,42 @@ // Add more forwarding functions to the m_data std::vector as needed. namespace vcpkg { - template + template> struct SortedVector { using size_type = typename std::vector::size_type; using iterator = typename std::vector::const_iterator; - SortedVector() = default; + SortedVector() : m_data(), m_comp() { } SortedVector(const SortedVector&) = default; SortedVector(SortedVector&&) = default; SortedVector& operator=(const SortedVector&) = default; SortedVector& operator=(SortedVector&&) = default; - explicit SortedVector(const std::vector& data) : m_data(data) { sort_uniqueify(std::less<>{}); } - - template - SortedVector(const std::vector& data, Compare comp) : m_data(data) + explicit SortedVector(const std::vector& data) : m_data(data), m_comp() { sort_uniqueify(); } + explicit SortedVector(const std::vector& data, Compare comp) : m_data(data), m_comp(comp) { - sort_uniqueify(comp); + sort_uniqueify(); } - - explicit SortedVector(std::vector&& data) : m_data(std::move(data)) { sort_uniqueify(std::less<>{}); } - - template - explicit SortedVector(std::vector&& data, Compare comp) : m_data(std::move(data)) + explicit SortedVector(std::vector&& data) : m_data(std::move(data)), m_comp() { sort_uniqueify(); } + explicit SortedVector(std::vector&& data, Compare comp) : m_data(std::move(data)), m_comp(comp) { - sort_uniqueify(comp); + sort_uniqueify(); } template - explicit SortedVector(InIt first, InIt last) : m_data(first, last) + explicit SortedVector(InIt first, InIt last) : m_data(first, last), m_comp() { - sort_uniqueify(std::less<>{}); + sort_uniqueify(); } - template - explicit SortedVector(InIt first, InIt last, Compare comp) : m_data(first, last) + template + explicit SortedVector(InIt first, InIt last, Compare comp) : m_data(first, last), m_comp(comp) { - sort_uniqueify(comp); + sort_uniqueify(); } - SortedVector(std::initializer_list elements) : m_data(elements) { sort_uniqueify(std::less<>{}); } + SortedVector(std::initializer_list elements) : m_data(elements), m_comp() { sort_uniqueify(); } iterator begin() const { return this->m_data.cbegin(); } @@ -67,36 +62,30 @@ namespace vcpkg bool contains(const Ty& element) const { return std::binary_search(m_data.begin(), m_data.end(), element); } - void append(const SortedVector& other) { append(other, std::less<>{}); } - - template - void append(const SortedVector& other, Compare comp) + void append(const SortedVector& other) { // This could use a more efficient merge algorithm than inplace_merge with an understanding that we will // allocate the whole result if perf becomes a problem auto merge_point = m_data.insert(m_data.end(), other.begin(), other.end()); - std::inplace_merge(m_data.begin(), merge_point, m_data.end(), comp); - uniqueify(comp); + std::inplace_merge(m_data.begin(), merge_point, m_data.end(), m_comp); + uniqueify(); } - void append(SortedVector&& other) { append(std::move(other), std::less<>{}); } - - template - void append(SortedVector&& other, Compare comp) + void append(SortedVector&& other) { auto merge_point = m_data.insert( m_data.end(), std::make_move_iterator(other.begin()), std::make_move_iterator(other.end())); - std::inplace_merge(m_data.begin(), merge_point, m_data.end(), comp); - uniqueify(comp); + std::inplace_merge(m_data.begin(), merge_point, m_data.end(), m_comp); + uniqueify(); } friend bool operator==(const SortedVector& lhs, const SortedVector& rhs) { return lhs.m_data == rhs.m_data; } friend bool operator!=(const SortedVector& lhs, const SortedVector& rhs) { return lhs.m_data != rhs.m_data; } private: - template - void uniqueify(Compare comp) + void uniqueify() { + Compare comp = m_comp; m_data.erase(std::unique(m_data.begin(), m_data.end(), [comp](const Ty& lhs, const Ty& rhs) { @@ -105,18 +94,17 @@ namespace vcpkg }), m_data.end()); } - - template - void sort_uniqueify(Compare comp) + void sort_uniqueify() { - if (!std::is_sorted(m_data.begin(), m_data.end(), comp)) + if (!std::is_sorted(m_data.begin(), m_data.end(), m_comp)) { - std::sort(m_data.begin(), m_data.end(), comp); + std::sort(m_data.begin(), m_data.end(), m_comp); } - uniqueify(comp); + uniqueify(); } std::vector m_data; + Compare m_comp; }; } diff --git a/src/vcpkg-test/update.cpp b/src/vcpkg-test/update.cpp index 5db1125994..c3561b6bb5 100644 --- a/src/vcpkg-test/update.cpp +++ b/src/vcpkg-test/update.cpp @@ -27,8 +27,8 @@ TEST_CASE ("find outdated packages basic", "[update]") map.emplace("a", SourceControlFileAndLocation{std::move(scf), ""}); PortFileProvider::MapPortFileProvider provider(map); - auto pkgs = SortedVector(Update::find_outdated_packages(provider, status_db), - &OutdatedPackage::compare_by_name); + auto pkgs = SortedVector( + Update::find_outdated_packages(provider, status_db), &OutdatedPackage::compare_by_name); REQUIRE(pkgs.size() == 1); REQUIRE(pkgs[0].version_diff.left.to_string() == "2"); @@ -51,8 +51,8 @@ TEST_CASE ("find outdated packages features", "[update]") map.emplace("a", SourceControlFileAndLocation{std::move(scf), ""}); PortFileProvider::MapPortFileProvider provider(map); - auto pkgs = SortedVector(Update::find_outdated_packages(provider, status_db), - &OutdatedPackage::compare_by_name); + auto pkgs = SortedVector( + Update::find_outdated_packages(provider, status_db), &OutdatedPackage::compare_by_name); REQUIRE(pkgs.size() == 1); REQUIRE(pkgs[0].version_diff.left.to_string() == "2"); @@ -77,8 +77,8 @@ TEST_CASE ("find outdated packages features 2", "[update]") map.emplace("a", SourceControlFileAndLocation{std::move(scf), ""}); PortFileProvider::MapPortFileProvider provider(map); - auto pkgs = SortedVector(Update::find_outdated_packages(provider, status_db), - &OutdatedPackage::compare_by_name); + auto pkgs = SortedVector( + Update::find_outdated_packages(provider, status_db), &OutdatedPackage::compare_by_name); REQUIRE(pkgs.size() == 1); REQUIRE(pkgs[0].version_diff.left.to_string() == "2"); @@ -98,8 +98,8 @@ TEST_CASE ("find outdated packages none", "[update]") map.emplace("a", SourceControlFileAndLocation{std::move(scf), ""}); PortFileProvider::MapPortFileProvider provider(map); - auto pkgs = SortedVector(Update::find_outdated_packages(provider, status_db), - &OutdatedPackage::compare_by_name); + auto pkgs = SortedVector( + Update::find_outdated_packages(provider, status_db), &OutdatedPackage::compare_by_name); REQUIRE(pkgs.size() == 0); } diff --git a/src/vcpkg/update.cpp b/src/vcpkg/update.cpp index 0b2c179645..180abc7e9b 100644 --- a/src/vcpkg/update.cpp +++ b/src/vcpkg/update.cpp @@ -70,8 +70,8 @@ namespace vcpkg::Update PortFileProvider::PathsPortFileProvider provider(paths, args.overlay_ports); - const auto outdated_packages = SortedVector(find_outdated_packages(provider, status_db), - &OutdatedPackage::compare_by_name); + const auto outdated_packages = SortedVector( + find_outdated_packages(provider, status_db), &OutdatedPackage::compare_by_name); if (outdated_packages.empty()) { diff --git a/src/vcpkg/visualstudio.cpp b/src/vcpkg/visualstudio.cpp index fb6da70fd8..c9e8a70cc7 100644 --- a/src/vcpkg/visualstudio.cpp +++ b/src/vcpkg/visualstudio.cpp @@ -210,8 +210,8 @@ namespace vcpkg::VisualStudio std::vector& paths_examined = ret.paths_examined; std::vector& found_toolsets = ret.toolsets; - const SortedVector sorted{get_visual_studio_instances_internal(fs), - VisualStudioInstance::preferred_first_comparator}; + const SortedVector sorted{ + get_visual_studio_instances_internal(fs), VisualStudioInstance::preferred_first_comparator}; const bool v140_is_available = Util::find_if(sorted, [&](const VisualStudioInstance& vs_instance) { return vs_instance.major_version() == "14"; From a3e92afa10712885ead03c6eca07d9be63d20403 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 23 Mar 2022 15:30:03 -0700 Subject: [PATCH 20/23] Localize --allow-unexpected-passing can only be used if a baseline is provided via --ci-baseline error. --- locales/messages.en.json | 1 + locales/messages.json | 1 + src/vcpkg/base/downloads.cpp | 9 ++++----- src/vcpkg/base/parse.cpp | 7 ++++--- src/vcpkg/commands.ci.cpp | 24 ++++++++++++++---------- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/locales/messages.en.json b/locales/messages.en.json index 7cd39e9ea9..eda81d97de 100644 --- a/locales/messages.en.json +++ b/locales/messages.en.json @@ -25,6 +25,7 @@ "ChecksLineInfo": "{vcpkg_line_info}: ", "ChecksUnreachableCode": "unreachable code was reached", "ChecksUpdateVcpkg": "updating vcpkg by rerunning bootstrap-vcpkg may resolve this failure.", + "CiBaselineAllowUnexpectedPassingRequiresBaseline": "--allow-unexpected-passing can only be used if a baseline is provided via --ci-baseline.", "CiBaselineRegression": "REGRESSION: {spec} failed with {build_result}. If expected, add {spec}=fail to {path}.", "CiBaselineRegressionHeader": "REGRESSIONS:", "CiBaselineUnexpectedPass": "PASSING, REMOVE FROM FAIL LIST: {spec} ({path}).", diff --git a/locales/messages.json b/locales/messages.json index 3da80b3460..50b803ca4c 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -47,6 +47,7 @@ "_ChecksLineInfo.comment": "{Locked}", "ChecksUnreachableCode": "unreachable code was reached", "ChecksUpdateVcpkg": "updating vcpkg by rerunning bootstrap-vcpkg may resolve this failure.", + "CiBaselineAllowUnexpectedPassingRequiresBaseline": "--allow-unexpected-passing can only be used if a baseline is provided via --ci-baseline.", "CiBaselineRegression": "REGRESSION: {spec} failed with {build_result}. If expected, add {spec}=fail to {path}.", "_CiBaselineRegression.comment": "example of {spec} is 'zlib:x64-windows'.\nexample of {build_result} is 'One of the BuildResultXxx messages (such as BuildResultSucceeded/SUCCEEDED)'.\nexample of {path} is '/foo/bar'.\n", "CiBaselineRegressionHeader": "REGRESSIONS:", diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 0148654744..97ad9e84e7 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -311,11 +311,10 @@ namespace vcpkg if (out->size() != start_size + urls.size()) { - Checks::msg_exit_with_message(VCPKG_LINE_INFO, - msg::format(msg::msgErrorMessage) - .append(msg::format(msgCurlReportedUnexpectedResults, - msg::command_line = cmd.command_line(), - msg::actual = Strings::join("\n", lines)))); + Checks::msg_exit_with_error(VCPKG_LINE_INFO, + msgCurlReportedUnexpectedResults, + msg::command_line = cmd.command_line(), + msg::actual = Strings::join("\n", lines)); } } std::vector url_heads(View urls, View headers) diff --git a/src/vcpkg/base/parse.cpp b/src/vcpkg/base/parse.cpp index 24b43f358b..7b352e3f82 100644 --- a/src/vcpkg/base/parse.cpp +++ b/src/vcpkg/base/parse.cpp @@ -112,9 +112,10 @@ namespace vcpkg Checks::msg_exit_with_message(VCPKG_LINE_INFO, LocalizedString::from_raw(error->format())); } - Checks::msg_check_exit(VCPKG_LINE_INFO, - warnings.empty(), - msg::format(msg::msgErrorMessage).append(msg::format(msgWarningsTreatedAsErrors))); + if (!warnings.empty()) + { + Checks::msg_exit_with_error(VCPKG_LINE_INFO, msgWarningsTreatedAsErrors); + } } ParserBase::ParserBase(StringView text, StringView origin, TextRowCol init_rowcol) diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index be99c3c965..1b2b27db23 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -96,6 +96,12 @@ namespace (msg::spec, msg::path), "", "PASSING, REMOVE FROM FAIL LIST: {spec} ({path})."); + + DECLARE_AND_REGISTER_MESSAGE( + CiBaselineAllowUnexpectedPassingRequiresBaseline, + (), + "", + "--allow-unexpected-passing can only be used if a baseline is provided via --ci-baseline."); } namespace vcpkg::Commands::CI @@ -537,7 +543,14 @@ namespace vcpkg::Commands::CI auto baseline_iter = settings.find(OPTION_CI_BASELINE); const bool allow_unexpected_passing = Util::Sets::contains(options.switches, OPTION_ALLOW_UNEXPECTED_PASSING); SortedVector expected_failures; - if (baseline_iter != settings.end()) + if (baseline_iter == settings.end()) + { + if (allow_unexpected_passing) + { + Checks::msg_exit_with_error(VCPKG_LINE_INFO, msgCiBaselineAllowUnexpectedPassingRequiresBaseline); + } + } + else { const auto& ci_baseline_file_name = baseline_iter->second; const auto ci_baseline_file_contents = @@ -547,15 +560,6 @@ namespace vcpkg::Commands::CI ci_parse_messages.exit_if_errors_or_warnings(ci_baseline_file_name); expected_failures = parse_and_apply_ci_baseline(lines, exclusions_map); } - else - { - Checks::check_exit(VCPKG_LINE_INFO, - !allow_unexpected_passing, - Strings::concat("--", - OPTION_ALLOW_UNEXPECTED_PASSING, - " can only be used if a ci baseline is provided via --", - OPTION_CI_BASELINE)); - } auto skipped_cascade_count = parse_skipped_cascade_count(settings); From 463db01b2700ae3f1202f0b5c2b2935c5ab953cf Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 23 Mar 2022 15:32:20 -0700 Subject: [PATCH 21/23] Revert ref qualification of extract_Xxx functions. --- include/vcpkg/base/parse.h | 4 ++-- src/vcpkg/base/json.cpp | 4 ++-- src/vcpkg/platform-expression.cpp | 2 +- src/vcpkg/sourceparagraph.cpp | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/vcpkg/base/parse.h b/include/vcpkg/base/parse.h index 7f4d18bc2e..73a2fea18e 100644 --- a/include/vcpkg/base/parse.h +++ b/include/vcpkg/base/parse.h @@ -133,10 +133,10 @@ namespace vcpkg void add_warning(LocalizedString&& message, const SourceLoc& loc); const ParseError* get_error() const { return m_messages.error.get(); } - std::unique_ptr extract_error() && { return std::move(m_messages.error); } + std::unique_ptr extract_error() { return std::move(m_messages.error); } const ParseMessages& messages() const { return m_messages; } - ParseMessages extract_messages() && { return std::move(m_messages); } + ParseMessages&& extract_messages() { return std::move(m_messages); } private: Unicode::Utf8Decoder m_it; diff --git a/src/vcpkg/base/json.cpp b/src/vcpkg/base/json.cpp index 4701749d20..6c41f07bff 100644 --- a/src/vcpkg/base/json.cpp +++ b/src/vcpkg/base/json.cpp @@ -994,11 +994,11 @@ namespace vcpkg::Json if (!parser.at_eof()) { parser.add_error("Unexpected character; expected EOF"); - return std::move(parser).extract_error(); + return parser.extract_error(); } else if (parser.get_error()) { - return std::move(parser).extract_error(); + return parser.extract_error(); } else { diff --git a/src/vcpkg/platform-expression.cpp b/src/vcpkg/platform-expression.cpp index fab65910a1..bbfbf8f117 100644 --- a/src/vcpkg/platform-expression.cpp +++ b/src/vcpkg/platform-expression.cpp @@ -630,7 +630,7 @@ namespace vcpkg::PlatformExpression ExpressionParser parser(expression, multiple_binary_operators); auto res = parser.parse(); - if (auto p = std::move(parser).extract_error()) + if (auto p = parser.extract_error()) { return p->format(); } diff --git a/src/vcpkg/sourceparagraph.cpp b/src/vcpkg/sourceparagraph.cpp index 9f381430e8..3c253a8848 100644 --- a/src/vcpkg/sourceparagraph.cpp +++ b/src/vcpkg/sourceparagraph.cpp @@ -1008,7 +1008,7 @@ namespace vcpkg { auto parser = SpdxLicenseExpressionParser(sv, ""); auto result = parser.parse(); - messages = std::move(parser).extract_messages(); + messages = parser.extract_messages(); return result; } From 19ae76dad6fecfb458764416a8346ddde836e6b6 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 23 Mar 2022 15:48:08 -0700 Subject: [PATCH 22/23] Simplify try_match_keyword. --- src/vcpkg/base/parse.cpp | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/src/vcpkg/base/parse.cpp b/src/vcpkg/base/parse.cpp index 7b352e3f82..17bb5e5b2e 100644 --- a/src/vcpkg/base/parse.cpp +++ b/src/vcpkg/base/parse.cpp @@ -160,34 +160,28 @@ namespace vcpkg bool ParserBase::try_match_keyword(StringView keyword_content) { - auto first1 = m_it; - const auto last1 = first1.end(); - - auto first2 = keyword_content.begin(); - const auto last2 = keyword_content.end(); - - for (;;) + auto encoded = m_it; + // check that the encoded stream matches the keyword: + for (const char ch : keyword_content) { - if (first2 == last2) + if (encoded.is_eof() || *encoded != static_cast(ch)) { - if (first1 == last1 || is_whitespace(*first1)) - { - m_it = first1; - m_column += static_cast(keyword_content.size()); - return true; - } - return false; } - if (first1 == last1 || *first1 != static_cast(*first2)) - { - return false; - } + ++encoded; + } - ++first1; - ++first2; + // whole keyword matched, now check for a word boundary: + if (!encoded.is_eof() && !is_whitespace(*encoded)) + { + return false; } + + // success + m_it = encoded; + m_column += static_cast(keyword_content.size()); + return true; } char32_t ParserBase::next() From 319ab06f496a203cc1bedc9de5e9eaaf064b9aff Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Thu, 24 Mar 2022 14:09:16 -0700 Subject: [PATCH 23/23] Add more tests requested by Robert. --- src/vcpkg-test/ci-baseline.cpp | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/vcpkg-test/ci-baseline.cpp b/src/vcpkg-test/ci-baseline.cpp index 335494abce..736a7ebd44 100644 --- a/src/vcpkg-test/ci-baseline.cpp +++ b/src/vcpkg-test/ci-baseline.cpp @@ -211,6 +211,11 @@ TEST_CASE ("Parse Errors", "[ci-baseline]") ^ )"); + check_error("hello\n:", R"(test:1:6: error: expected ':' here + on expression: hello + ^ +)"); + check_error("?example:x64-windows=fail", R"(test:1:1: error: expected a port name here on expression: ?example:x64-windows=fail ^ @@ -221,6 +226,11 @@ TEST_CASE ("Parse Errors", "[ci-baseline]") ^ )"); + check_error("x64-windows:\nport:x64-windows=skip", R"(test:1:13: error: expected a triplet name here + on expression: x64-windows: + ^ +)"); + check_error("x64-windows:#", R"(test:1:13: error: expected a triplet name here on expression: x64-windows:# ^ @@ -233,6 +243,16 @@ TEST_CASE ("Parse Errors", "[ci-baseline]") )"); // clang-format on + check_error("port:x64-windows\n=fail", R"(test:1:17: error: expected '=' here + on expression: port:x64-windows + ^ +)"); + + check_error("example:x64-windows = \n fail", R"(test:1:26: error: expected 'fail' or 'skip' here + on expression: example:x64-windows = + ^ +)"); + check_error("example:x64-windows = pass", R"(test:1:28: error: expected 'fail' or 'skip' here on expression: example:x64-windows = pass ^ @@ -250,6 +270,12 @@ TEST_CASE ("Parse Errors", "[ci-baseline]") ^ )"); + check_error("example:x64-windows = fail example:x64-windows = fail", + R"(test:1:33: error: unrecognizable baseline entry; expected 'port:triplet=(fail|skip)' + on expression: example:x64-windows = fail example:x64-windows = fail + ^ +)"); + check_error("example:x64-windows = fail # extra stuff\n" "example:x64-uwp=skip extra stuff\n", R"(test:2:22: error: unrecognizable baseline entry; expected 'port:triplet=(fail|skip)'