[vcpkg] Implement versions db generator (#13777)

scripts/generatePortVersionsDb.py

@@ -0,0 +1,71 @@
+import os
+import os.path
+import sys
+import subprocess
+import json
+import time
+
+from subprocess import CalledProcessError
+from json.decoder import JSONDecodeError
+from pathlib import Path
+
+
+SCRIPT_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
+
+
+def get_current_git_ref():
+    output = subprocess.run(['git.exe', '-C', SCRIPT_DIRECTORY, 'rev-parse', '--verify', 'HEAD'], 
+        capture_output=True, 
+        encoding='utf-8')
+    if output.returncode == 0:
+        return output.stdout.strip()
+    print(f"Failed to get git ref:", output.stderr.strip(), file=sys.stderr)
+    return None
+
+
+def generate_port_versions_db(ports_path, db_path, revision):
+    start_time = time.time()
+    port_names = [item for item in os.listdir(ports_path) if os.path.isdir(os.path.join(ports_path, item))]
+    total_count = len(port_names)
+    for counter, port_name in enumerate(port_names):
+        containing_dir = os.path.join(db_path, f'{port_name[0]}-')
+        os.makedirs(containing_dir, exist_ok=True)
+        output_filepath = os.path.join(containing_dir, f'{port_name}.json')
+        if not os.path.exists(output_filepath):            
+            output = subprocess.run(
+                [os.path.join(SCRIPT_DIRECTORY, '../vcpkg'), 'x-history', port_name, '--x-json'], 
+                capture_output=True, encoding='utf-8')
+            if output.returncode == 0:
+                try:
+                    versions_object = json.loads(output.stdout)
+                    with open(output_filepath, 'w') as output_file:
+                        json.dump(versions_object, output_file)
+                except JSONDecodeError:
+                    print(f'Maformed JSON from vcpkg x-history {port_name}: ', output.stdout.strip(), file=sys.stderr)
+            else:
+                print(f'x-history {port_name} failed: ', output.stdout.strip(), file=sys.stderr)
+        # This should be replaced by a progress bar
+        if counter > 0 and counter % 100 == 0:
+            elapsed_time = time.time() - start_time
+            print(f'Processed {counter} out of {total_count}. Elapsed time: {elapsed_time:.2f} seconds')
+    rev_file = os.path.join(db_path, revision)
+    Path(rev_file).touch()
+    elapsed_time = time.time() - start_time
+    print(f'Processed {total_count} total ports. Elapsed time: {elapsed_time:.2f} seconds')
+
+
+def main(ports_path, db_path):
+    revision = get_current_git_ref()
+    if not revision:
+        print('Couldn\'t fetch current Git revision', file=sys.stderr)
+        sys.exit(1)
+    rev_file = os.path.join(db_path, revision)
+    if os.path.exists(rev_file):
+        print(f'Database files already exist for commit {revision}')
+        sys.exit(0)
+    generate_port_versions_db(ports_path=ports_path, db_path=db_path, revision=revision)
+
+
+if __name__ == "__main__":
+    main(ports_path=os.path.join(SCRIPT_DIRECTORY, '../ports'), 
+        db_path=os.path.join(SCRIPT_DIRECTORY, '../port_versions'))

toolsrc/include/vcpkg/paragraphs.h

@@ -17,12 +17,18 @@ namespace vcpkg::Paragraphs
 
     ExpectedS<Paragraph> parse_single_paragraph(const std::string& str, const std::string& origin);
     ExpectedS<Paragraph> get_single_paragraph(const Files::Filesystem& fs, const fs::path& control_path);
+
     ExpectedS<std::vector<Paragraph>> get_paragraphs(const Files::Filesystem& fs, const fs::path& control_path);
+    ExpectedS<std::vector<Paragraph>> get_paragraphs_text(const std::string& text, const std::string& origin);
+
     ExpectedS<std::vector<Paragraph>> parse_paragraphs(const std::string& str, const std::string& origin);
 
     bool is_port_directory(const Files::Filesystem& fs, const fs::path& path);
 
     Parse::ParseExpected<SourceControlFile> try_load_port(const Files::Filesystem& fs, const fs::path& path);
+    Parse::ParseExpected<SourceControlFile> try_load_port_text(const std::string& text,
+                                                               const std::string& origin,
+                                                               bool is_manifest);
 
     ExpectedS<BinaryControlFile> try_load_cached_package(const VcpkgPaths& paths, const PackageSpec& spec);
 

toolsrc/include/vcpkg/sourceparagraph.h

@@ -91,11 +91,14 @@ namespace vcpkg
             return ret;
         }
 
+        static Parse::ParseExpected<SourceControlFile> parse_manifest_object(const std::string& origin,
+                                                                             const Json::Object& object);
+
         static Parse::ParseExpected<SourceControlFile> parse_manifest_file(const fs::path& path_to_manifest,
                                                                            const Json::Object& object);
 
         static Parse::ParseExpected<SourceControlFile> parse_control_file(
-            const fs::path& path_to_control, std::vector<Parse::Paragraph>&& control_paragraphs);
+            const std::string& origin, std::vector<Parse::Paragraph>&& control_paragraphs);
 
         // Always non-null in non-error cases
         std::unique_ptr<SourceParagraph> core_paragraph;

toolsrc/include/vcpkg/versions.h

@@ -1,14 +1,12 @@
 #pragma once
 
-#include <vcpkg/fwd/vcpkgpaths.h>
-
 namespace vcpkg::Versions
 {
     enum class Scheme
     {
-        String,
         Relaxed,
         Semver,
-        Date
+        Date,
+        String
     };
 }

toolsrc/src/vcpkg-test/binarycaching.cpp

@@ -75,7 +75,7 @@ Build-Depends: bzip
 )",
                                              "<testdata>");
     REQUIRE(pghs.has_value());
-    auto maybe_scf = SourceControlFile::parse_control_file(fs::path(), std::move(*pghs.get()));
+    auto maybe_scf = SourceControlFile::parse_control_file(fs::u8string(fs::path()), std::move(*pghs.get()));
     REQUIRE(maybe_scf.has_value());
     SourceControlFileLocation scfl{std::move(*maybe_scf.get()), fs::path()};
 
@@ -255,7 +255,7 @@ Description: a spiffy compression library wrapper
 )",
                                              "<testdata>");
     REQUIRE(pghs.has_value());
-    auto maybe_scf = SourceControlFile::parse_control_file(fs::path(), std::move(*pghs.get()));
+    auto maybe_scf = SourceControlFile::parse_control_file(fs::u8string(fs::path()), std::move(*pghs.get()));
     REQUIRE(maybe_scf.has_value());
     SourceControlFileLocation scfl{std::move(*maybe_scf.get()), fs::path()};
     plan.install_actions.push_back(Dependencies::InstallPlanAction());
@@ -278,7 +278,7 @@ Description: a spiffy compression library wrapper
 )",
                                               "<testdata>");
     REQUIRE(pghs2.has_value());
-    auto maybe_scf2 = SourceControlFile::parse_control_file(fs::path(), std::move(*pghs2.get()));
+    auto maybe_scf2 = SourceControlFile::parse_control_file(fs::u8string(fs::path()), std::move(*pghs2.get()));
     REQUIRE(maybe_scf2.has_value());
     SourceControlFileLocation scfl2{std::move(*maybe_scf2.get()), fs::path()};
     plan.install_actions.push_back(Dependencies::InstallPlanAction());

toolsrc/src/vcpkg-test/manifests.cpp

@@ -331,9 +331,9 @@ TEST_CASE ("Serialize all the ports", "[manifests]")
             auto pghs = Paragraphs::parse_paragraphs(contents, fs::u8string(control));
             REQUIRE(pghs);
 
-            scfs.push_back(std::move(
-                *SourceControlFile::parse_control_file(control, std::move(pghs).value_or_exit(VCPKG_LINE_INFO))
-                     .value_or_exit(VCPKG_LINE_INFO)));
+            scfs.push_back(std::move(*SourceControlFile::parse_control_file(
+                                          fs::u8string(control), std::move(pghs).value_or_exit(VCPKG_LINE_INFO))
+                                          .value_or_exit(VCPKG_LINE_INFO)));
         }
         else if (fs.exists(manifest))
         {

toolsrc/src/vcpkg-test/paragraph.cpp

@@ -51,6 +51,38 @@ TEST_CASE ("SourceParagraph construct minimum", "[paragraph]")
     REQUIRE(pgh.core_paragraph->dependencies.size() == 0);
 }
 
+TEST_CASE ("SourceParagraph construct invalid", "[paragraph]")
+{
+    auto m_pgh = test_parse_control_file({{
+        {"Source", "zlib"},
+        {"Version", "1.2.8"},
+        {"Build-Depends", "1.2.8"},
+    }});
+
+    REQUIRE(!m_pgh.has_value());
+
+    m_pgh = test_parse_control_file({{
+        {"Source", "zlib"},
+        {"Version", "1.2.8"},
+        {"Default-Features", "1.2.8"},
+    }});
+
+    REQUIRE(!m_pgh.has_value());
+
+    m_pgh = test_parse_control_file({
+        {
+            {"Source", "zlib"},
+            {"Version", "1.2.8"},
+        },
+        {
+            {"Feature", "a"},
+            {"Build-Depends", "1.2.8"},
+        },
+    });
+
+    REQUIRE(!m_pgh.has_value());
+}
+
 TEST_CASE ("SourceParagraph construct maximum", "[paragraph]")
 {
     auto m_pgh = test_parse_control_file({{
@@ -76,6 +108,24 @@ TEST_CASE ("SourceParagraph construct maximum", "[paragraph]")
     REQUIRE(pgh.core_paragraph->default_features[0] == "df");
 }
 
+TEST_CASE ("SourceParagraph construct feature", "[paragraph]")
+{
+    auto m_pgh = test_parse_control_file({
+        {
+            {"Source", "s"},
+            {"Version", "v"},
+        },
+        {{"Feature", "f"}, {"Description", "d2"}, {"Build-Depends", "bd2"}},
+    });
+    REQUIRE(m_pgh.has_value());
+    auto& pgh = **m_pgh.get();
+
+    REQUIRE(pgh.feature_paragraphs.size() == 1);
+    REQUIRE(pgh.feature_paragraphs[0]->name == "f");
+    REQUIRE(pgh.feature_paragraphs[0]->description == std::vector<std::string>{"d2"});
+    REQUIRE(pgh.feature_paragraphs[0]->dependencies.size() == 1);
+}
+
 TEST_CASE ("SourceParagraph two dependencies", "[paragraph]")
 {
     auto m_pgh = test_parse_control_file({{

toolsrc/src/vcpkg/commands.format-manifest.cpp

@@ -80,8 +80,8 @@ namespace
                            paragraphs.error());
             return {};
         }
-        auto scf_res =
-            SourceControlFile::parse_control_file(control_path, std::move(paragraphs).value_or_exit(VCPKG_LINE_INFO));
+        auto scf_res = SourceControlFile::parse_control_file(fs::u8string(control_path),
+                                                             std::move(paragraphs).value_or_exit(VCPKG_LINE_INFO));
         if (!scf_res)
         {
             System::printf(System::Color::error, "Failed to parse control file: %s\n", control_path_string);