vcpkg ci: Parse ci.baseline.txt to determine regressions

include/vcpkg-test/util.h

@@ -5,7 +5,9 @@
 #include <vcpkg/base/files.h>
 #include <vcpkg/base/messages.h>
 #include <vcpkg/base/pragmas.h>
+#include <vcpkg/base/sortedvector.h>
 
+#include <vcpkg/packagespec.h>
 #include <vcpkg/statusparagraph.h>
 
 #include <iomanip>
@@ -55,10 +57,18 @@ namespace Catch
     {
         static const std::string convert(const vcpkg::LocalizedString& value) { return "LL\"" + value.data() + "\""; }
     };
+
+    template<>
+    struct StringMaker<vcpkg::PackageSpec>
+    {
+        static const std::string convert(const vcpkg::PackageSpec& value) { return value.to_string(); }
+    };
 }
 
 namespace vcpkg
 {
+    inline std::ostream& operator<<(std::ostream& os, const PackageSpec& value) { return os << value.to_string(); }
+
     inline std::ostream& operator<<(std::ostream& os, const LocalizedString& value)
     {
         return os << "LL" << std::quoted(value.data());

include/vcpkg/base/parse.h

@@ -58,6 +58,9 @@ namespace vcpkg
     {
         std::unique_ptr<ParseError> error;
         std::vector<ParseMessage> warnings;
+
+        void exit_if_errors_or_warnings(StringView origin) const;
+        bool good() const { return !error && warnings.empty(); }
     };
 
     struct ParserBase
@@ -84,42 +87,35 @@ namespace vcpkg
         }
         static constexpr bool is_word_char(char32_t ch) { return is_alphanum(ch) || ch == '_'; }
 
-        StringView skip_whitespace() { return match_zero_or_more(is_whitespace); }
-        StringView skip_tabs_spaces()
-        {
-            return match_zero_or_more([](char32_t ch) { return ch == ' ' || ch == '\t'; });
-        }
-        void skip_to_eof() { m_it = m_it.end(); }
-        void skip_newline()
-        {
-            if (cur() == '\r') next();
-            if (cur() == '\n') next();
-        }
-        void skip_line()
-        {
-            match_until(is_lineend);
-            skip_newline();
-        }
+        StringView skip_whitespace();
+        StringView skip_tabs_spaces();
+        void skip_to_eof();
+        void skip_newline();
+        void skip_line();
 
         template<class Pred>
-        StringView match_zero_or_more(Pred p)
+        StringView match_while(Pred p)
         {
             const char* start = m_it.pointer_to_current();
             auto ch = cur();
             while (ch != Unicode::end_of_file && p(ch))
+            {
                 ch = next();
+            }
+
             return {start, m_it.pointer_to_current()};
         }
+
         template<class Pred>
         StringView match_until(Pred p)
         {
-            const char* start = m_it.pointer_to_current();
-            auto ch = cur();
-            while (ch != Unicode::end_of_file && !p(ch))
-                ch = next();
-            return {start, m_it.pointer_to_current()};
+            return match_while([p](char32_t ch) { return !p(ch); });
         }
 
+        bool require_character(char ch);
+
+        bool try_match_keyword(StringView keyword_content);
+
         StringView text() const { return m_text; }
         Unicode::Utf8Decoder it() const { return m_it; }
         char32_t cur() const { return m_it == m_it.end() ? Unicode::end_of_file : *m_it; }
@@ -140,7 +136,7 @@ namespace vcpkg
         std::unique_ptr<ParseError> extract_error() { return std::move(m_messages.error); }
 
         const ParseMessages& messages() const { return m_messages; }
-        ParseMessages extract_messages() { return std::move(m_messages); }
+        ParseMessages&& extract_messages() { return std::move(m_messages); }
 
     private:
         Unicode::Utf8Decoder m_it;

include/vcpkg/base/sortedvector.h

@@ -1,36 +1,51 @@
 #pragma once
 
 #include <algorithm>
+#include <functional>
+#include <initializer_list>
+#include <iterator>
 #include <vector>
 
 // Add more forwarding functions to the m_data std::vector as needed.
 namespace vcpkg
 {
-    template<class T>
+    template<class Ty, class Compare = std::less<>>
     struct SortedVector
     {
-        using size_type = typename std::vector<T>::size_type;
-        using iterator = typename std::vector<T>::const_iterator;
+        using size_type = typename std::vector<Ty>::size_type;
+        using iterator = typename std::vector<Ty>::const_iterator;
 
-        SortedVector() : m_data() { }
+        SortedVector() : m_data(), m_comp() { }
+        SortedVector(const SortedVector&) = default;
+        SortedVector(SortedVector&&) = default;
+        SortedVector& operator=(const SortedVector&) = default;
+        SortedVector& operator=(SortedVector&&) = default;
 
-        explicit SortedVector(std::vector<T> v) : m_data(std::move(v))
+        explicit SortedVector(const std::vector<Ty>& data) : m_data(data), m_comp() { sort_uniqueify(); }
+        explicit SortedVector(const std::vector<Ty>& data, Compare comp) : m_data(data), m_comp(comp)
         {
-            if (!std::is_sorted(m_data.begin(), m_data.end()))
-            {
-                std::sort(m_data.begin(), m_data.end());
-            }
+            sort_uniqueify();
+        }
+        explicit SortedVector(std::vector<Ty>&& data) : m_data(std::move(data)), m_comp() { sort_uniqueify(); }
+        explicit SortedVector(std::vector<Ty>&& data, Compare comp) : m_data(std::move(data)), m_comp(comp)
+        {
+            sort_uniqueify();
         }
 
-        template<class Compare>
-        SortedVector(std::vector<T> v, Compare comp) : m_data(std::move(v))
+        template<class InIt>
+        explicit SortedVector(InIt first, InIt last) : m_data(first, last), m_comp()
         {
-            if (!std::is_sorted(m_data.cbegin(), m_data.cend(), comp))
-            {
-                std::sort(m_data.begin(), m_data.end(), comp);
-            }
+            sort_uniqueify();
         }
 
+        template<class InIt>
+        explicit SortedVector(InIt first, InIt last, Compare comp) : m_data(first, last), m_comp(comp)
+        {
+            sort_uniqueify();
+        }
+
+        SortedVector(std::initializer_list<Ty> elements) : m_data(elements), m_comp() { sort_uniqueify(); }
+
         iterator begin() const { return this->m_data.cbegin(); }
 
         iterator end() const { return this->m_data.cend(); }
@@ -43,9 +58,53 @@ namespace vcpkg
 
         size_type size() const { return this->m_data.size(); }
 
-        const T& operator[](int i) const { return this->m_data[i]; }
+        const Ty& operator[](std::size_t i) const { return this->m_data[i]; }
+
+        bool contains(const Ty& element) const { return std::binary_search(m_data.begin(), m_data.end(), element); }
+
+        void append(const SortedVector& other)
+        {
+            // This could use a more efficient merge algorithm than inplace_merge with an understanding that we will
+            // allocate the whole result if perf becomes a problem
+            auto merge_point = m_data.insert(m_data.end(), other.begin(), other.end());
+            std::inplace_merge(m_data.begin(), merge_point, m_data.end(), m_comp);
+            uniqueify();
+        }
+
+        void append(SortedVector&& other)
+        {
+            auto merge_point = m_data.insert(
+                m_data.end(), std::make_move_iterator(other.begin()), std::make_move_iterator(other.end()));
+            std::inplace_merge(m_data.begin(), merge_point, m_data.end(), m_comp);
+            uniqueify();
+        }
+
+        friend bool operator==(const SortedVector& lhs, const SortedVector& rhs) { return lhs.m_data == rhs.m_data; }
+        friend bool operator!=(const SortedVector& lhs, const SortedVector& rhs) { return lhs.m_data != rhs.m_data; }
 
     private:
-        std::vector<T> m_data;
+        void uniqueify()
+        {
+            Compare comp = m_comp;
+            m_data.erase(std::unique(m_data.begin(),
+                                     m_data.end(),
+                                     [comp](const Ty& lhs, const Ty& rhs) {
+                                         // note that we know !comp(rhs, lhs) because m_data is sorted
+                                         return !comp(lhs, rhs);
+                                     }),
+                         m_data.end());
+        }
+        void sort_uniqueify()
+        {
+            if (!std::is_sorted(m_data.begin(), m_data.end(), m_comp))
+            {
+                std::sort(m_data.begin(), m_data.end(), m_comp);
+            }
+
+            uniqueify();
+        }
+
+        std::vector<Ty> m_data;
+        Compare m_comp;
     };
 }

include/vcpkg/ci-baseline.h

@@ -0,0 +1,58 @@
+#pragma once
+#include <vcpkg/fwd/ci-baseline.h>
+
+#include <vcpkg/base/expected.h>
+#include <vcpkg/base/sortedvector.h>
+#include <vcpkg/base/view.h>
+
+#include <vcpkg/packagespec.h>
+#include <vcpkg/triplet.h>
+
+#include <initializer_list>
+#include <string>
+#include <vector>
+
+namespace vcpkg
+{
+    struct CiBaselineLine
+    {
+        std::string port_name;
+        Triplet triplet;
+        CiBaselineState state;
+
+        friend bool operator==(const CiBaselineLine& lhs, const CiBaselineLine& rhs)
+        {
+            return lhs.port_name == rhs.port_name && lhs.triplet == rhs.triplet && lhs.state == rhs.state;
+        }
+
+        friend bool operator!=(const CiBaselineLine& lhs, const CiBaselineLine& rhs) { return !(lhs == rhs); }
+    };
+
+    struct TripletExclusions
+    {
+        Triplet triplet;
+        SortedVector<std::string> exclusions;
+
+        TripletExclusions(const Triplet& triplet);
+        TripletExclusions(const Triplet& triplet, SortedVector<std::string>&& exclusions);
+    };
+
+    struct ExclusionsMap
+    {
+        std::vector<TripletExclusions> triplets;
+
+        void insert(Triplet triplet);
+        void insert(Triplet triplet, SortedVector<std::string>&& exclusions);
+    };
+
+    struct ExclusionPredicate
+    {
+        const ExclusionsMap* data;
+
+        bool operator()(const PackageSpec& spec) const;
+    };
+
+    std::vector<CiBaselineLine> parse_ci_baseline(StringView text, StringView origin, ParseMessages& messages);
+
+    SortedVector<PackageSpec> parse_and_apply_ci_baseline(View<CiBaselineLine> lines, ExclusionsMap& exclusions_map);
+}

include/vcpkg/fwd/ci-baseline.h

@@ -0,0 +1,16 @@
+#pragma once
+
+namespace vcpkg
+{
+    struct CiBaseline;
+    struct CiBaselineLine;
+    struct TripletExclusions;
+    struct ExclusionsMap;
+    struct ExclusionPredicate;
+
+    enum class CiBaselineState
+    {
+        Skip,
+        Fail,
+    };
+}

include/vcpkg/packagespec.h

@@ -23,7 +23,8 @@ namespace vcpkg
     struct PackageSpec
     {
         PackageSpec() = default;
-        PackageSpec(std::string name, Triplet triplet) : m_name(std::move(name)), m_triplet(triplet) { }
+        PackageSpec(const std::string& name, Triplet triplet) : m_name(name), m_triplet(triplet) { }
+        PackageSpec(std::string&& name, Triplet triplet) : m_name(std::move(name)), m_triplet(triplet) { }
 
         const std::string& name() const;
 

locales/messages.en.json

@@ -25,6 +25,10 @@
   "ChecksLineInfo": "{vcpkg_line_info}: ",
   "ChecksUnreachableCode": "unreachable code was reached",
   "ChecksUpdateVcpkg": "updating vcpkg by rerunning bootstrap-vcpkg may resolve this failure.",
+  "CiBaselineAllowUnexpectedPassingRequiresBaseline": "--allow-unexpected-passing can only be used if a baseline is provided via --ci-baseline.",
+  "CiBaselineRegression": "REGRESSION: {spec} failed with {build_result}. If expected, add {spec}=fail to {path}.",
+  "CiBaselineRegressionHeader": "REGRESSIONS:",
+  "CiBaselineUnexpectedPass": "PASSING, REMOVE FROM FAIL LIST: {spec} ({path}).",
   "CmakeTargetLinkLibraries": "    target_link_libraries(main PRIVATE {list})",
   "CmakeTargetsExcluded": "    # note: {count} targets were omitted.",
   "CouldNotDeduceNugetIdAndVersion": "Could not deduce nuget id and version from filename: {path}",
@@ -45,10 +49,13 @@
   "ErrorRequirePackagesList": "Error: `vcpkg install` requires a list of packages to install in classic mode.",
   "ErrorRequirePackagesToInstall": "Error: No packages were listed for installation and no manifest was found.",
   "ErrorVcvarsUnsupported": "Error: in triplet {triplet}: Use of Visual Studio's Developer Prompt is unsupported on non-Windows hosts.\nDefine 'VCPKG_CMAKE_SYSTEM_NAME' or 'VCPKG_CHAINLOAD_TOOLCHAIN_FILE' in the triplet file.",
+  "ExpectedCharacterHere": "expected '{expected}' here",
+  "ExpectedFailOrSkip": "expected 'fail' or 'skip' here",
+  "ExpectedPortName": "expected a port name here",
+  "ExpectedTripletName": "expected a triplet name here",
   "FailedToProvisionCe": "Failed to provision vcpkg-ce.",
   "ForceSystemBinariesOnWeirdPlatforms": "Environment variable VCPKG_FORCE_SYSTEM_BINARIES must be set on arm, s390x, and ppc64le platforms.",
   "FormattedParseMessageExpression": "    on expression: {value}",
-  "FormattedParseMessageLocation": "{path}:{row}:{column}: ",
   "GenerateMsgErrorParsingFormatArgs": "error: parsing format string for {value}:",
   "GenerateMsgIncorrectComment": "message {value} has an incorrect comment:",
   "GenerateMsgNoArgumentValue": "    {{{value}}} was specified in a comment, but was not used in the message.",
@@ -87,6 +94,7 @@
   "ResultsLine": "    {spec}: {build_result}: {elapsed}",
   "SeeURL": "See {url} for more information.",
   "SuggestNewVersionScheme": "Use the version scheme \"{new_scheme}\" instead of \"{old_scheme}\" in port \"{package_name}\".\nUse `--{option}` to disable this check.",
+  "UnknownBaselineFileContent": "unrecognizable baseline entry; expected 'port:triplet=(fail|skip)'",
   "UnsupportedSystemName": "Error: Could not map VCPKG_CMAKE_SYSTEM_NAME '{system_name}' to a vcvarsall platform. Supported system names are '', 'Windows' and 'WindowsStore'.",
   "UnsupportedToolchain": "Error: in triplet {triplet}: Unable to find a valid toolchain combination.\n    The requested target architecture was {arch}\n    The selected Visual Studio instance is at {path}\n    The available toolchain combinations are {list}\n",
   "UsingManifestAt": "Using manifest file at {path}.",
@@ -105,5 +113,6 @@
   "VersionInvalidRelaxed": "`{version}` is not a valid relaxed version (semver with arbitrary numeric element count).",
   "VersionInvalidSemver": "`{version}` is not a valid semantic version, consult <https://semver.org>.",
   "VersionSpecMismatch": "error: Failed to load port because version specs did not match\n    Path: {path}\n    Expected: {expected_version}\n    Actual: {actual_version}",
-  "WarningMessage": "warning: "
+  "WarningMessage": "warning: ",
+  "WarningsTreatedAsErrors": "previous warnings being interpreted as errors"
 }