diff --git a/toolsrc/include/vcpkg/base/system.h b/toolsrc/include/vcpkg/base/system.h index 4172f0c5001e47..f119dbdc438f68 100644 --- a/toolsrc/include/vcpkg/base/system.h +++ b/toolsrc/include/vcpkg/base/system.h @@ -9,6 +9,8 @@ namespace vcpkg::System { Optional get_environment_variable(ZStringView varname) noexcept; + ExpectedS get_home_dir() noexcept; + Optional get_registry_string(void* base_hkey, StringView subkey, StringView valuename); enum class CPUArchitecture diff --git a/toolsrc/include/vcpkg/binarycaching.h b/toolsrc/include/vcpkg/binarycaching.h index 199b01accc03dc..69e3287d601985 100644 --- a/toolsrc/include/vcpkg/binarycaching.h +++ b/toolsrc/include/vcpkg/binarycaching.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -35,5 +36,8 @@ namespace vcpkg bool purge_tombstones) = 0; }; - std::unique_ptr create_archives_provider(); + ExpectedS> create_binary_provider_from_configs(const VcpkgPaths& paths, + View args); + ExpectedS> create_binary_provider_from_configs_pure(const std::string& env_string, + View args); } diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index d8e87f131695d2..560e856b9ffd50 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -34,6 +34,7 @@ namespace vcpkg::Build void perform_and_exit_ex(const FullPackageSpec& full_spec, const SourceControlFileLocation& scfl, const PortFileProvider::PathsPortFileProvider& provider, + IBinaryProvider& binaryprovider, const VcpkgPaths& paths); void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths, Triplet default_triplet); @@ -209,7 +210,7 @@ namespace vcpkg::Build ExtendedBuildResult build_package(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& config, - IBinaryProvider* binaries_provider, + IBinaryProvider& binaries_provider, const StatusParagraphs& status_db); enum class BuildPolicy diff --git a/toolsrc/include/vcpkg/install.h b/toolsrc/include/vcpkg/install.h index 1c42cc102c13aa..ad91bbb378281c 100644 --- a/toolsrc/include/vcpkg/install.h +++ b/toolsrc/include/vcpkg/install.h @@ -79,6 +79,7 @@ namespace vcpkg::Install const KeepGoing keep_going, const VcpkgPaths& paths, StatusParagraphs& status_db, + IBinaryProvider& binaryprovider, const CMakeVars::CMakeVarProvider& var_provider); extern const CommandStructure COMMAND_STRUCTURE; diff --git a/toolsrc/include/vcpkg/vcpkgcmdarguments.h b/toolsrc/include/vcpkg/vcpkgcmdarguments.h index e5b26c25afcbae..92584fc6a39d3f 100644 --- a/toolsrc/include/vcpkg/vcpkgcmdarguments.h +++ b/toolsrc/include/vcpkg/vcpkgcmdarguments.h @@ -91,6 +91,7 @@ namespace vcpkg std::unique_ptr triplet; std::unique_ptr> overlay_ports; std::unique_ptr> overlay_triplets; + std::vector binarysources; Optional debug = nullopt; Optional sendmetrics = nullopt; Optional printmetrics = nullopt; diff --git a/toolsrc/src/vcpkg-test/binaryconfigparser.cpp b/toolsrc/src/vcpkg-test/binaryconfigparser.cpp new file mode 100644 index 00000000000000..910d0836fb67ea --- /dev/null +++ b/toolsrc/src/vcpkg-test/binaryconfigparser.cpp @@ -0,0 +1,183 @@ +#include +#include + +using namespace vcpkg; + +#if defined(_WIN32) +#define ABSOLUTE_PATH "C:\\foo" +#else +#define ABSOLUTE_PATH "/foo" +#endif + +TEST_CASE ("BinaryConfigParser empty", "[binaryconfigparser]") +{ + auto parsed = create_binary_provider_from_configs_pure("", {}); + REQUIRE(parsed.has_value()); +} + +TEST_CASE ("BinaryConfigParser unacceptable provider", "[binaryconfigparser]") +{ + auto parsed = create_binary_provider_from_configs_pure("unacceptable", {}); + REQUIRE(!parsed.has_value()); +} + +TEST_CASE ("BinaryConfigParser files provider", "[binaryconfigparser]") +{ + { + auto parsed = create_binary_provider_from_configs_pure("files", {}); + REQUIRE(!parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("files,relative-path", {}); + REQUIRE(!parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("files,C:foo", {}); + REQUIRE(!parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH, {}); + REQUIRE(parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH ",nonsense", {}); + REQUIRE(!parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH ",upload", {}); + REQUIRE(parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH ",upload,extra", {}); + REQUIRE(!parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("files,,upload", {}); + REQUIRE(!parsed.has_value()); + } +} + +TEST_CASE ("BinaryConfigParser default provider", "[binaryconfigparser]") +{ + { + auto parsed = create_binary_provider_from_configs_pure("default", {}); + REQUIRE(parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("default,nonsense", {}); + REQUIRE(!parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("default,upload", {}); + REQUIRE(parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("default,upload,extra", {}); + REQUIRE(!parsed.has_value()); + } +} + +TEST_CASE ("BinaryConfigParser clear provider", "[binaryconfigparser]") +{ + { + auto parsed = create_binary_provider_from_configs_pure("clear", {}); + REQUIRE(parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("clear,upload", {}); + REQUIRE(!parsed.has_value()); + } +} + +TEST_CASE ("BinaryConfigParser multiple providers", "[binaryconfigparser]") +{ + { + auto parsed = create_binary_provider_from_configs_pure("clear;default", {}); + REQUIRE(parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("clear;default,upload", {}); + REQUIRE(parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("clear;default,upload;clear;clear", {}); + REQUIRE(parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("clear;files,relative;default", {}); + REQUIRE(!parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure(";;;clear;;;;", {}); + REQUIRE(parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure(";;;,;;;;", {}); + REQUIRE(!parsed.has_value()); + } +} + +TEST_CASE ("BinaryConfigParser escaping", "[binaryconfigparser]") +{ + { + auto parsed = create_binary_provider_from_configs_pure(";;;;;;;`", {}); + REQUIRE(!parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure(";;;;;;;`defaul`t", {}); + REQUIRE(parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH "`", {}); + REQUIRE(!parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH "`,", {}); + REQUIRE(parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH "``", {}); + REQUIRE(parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH "```", {}); + REQUIRE(!parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH "````", {}); + REQUIRE(parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH ",", {}); + REQUIRE(!parsed.has_value()); + } +} + +TEST_CASE ("BinaryConfigParser args", "[binaryconfigparser]") +{ + { + auto parsed = + create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH, std::vector{"clear"}); + REQUIRE(parsed.has_value()); + } + { + auto parsed = + create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH, std::vector{"clear;default"}); + REQUIRE(parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH, + std::vector{"clear;default,"}); + REQUIRE(!parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH, + std::vector{"clear", "clear;default,"}); + REQUIRE(!parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH, + std::vector{"clear", "clear"}); + REQUIRE(parsed.has_value()); + } +} diff --git a/toolsrc/src/vcpkg/base/system.cpp b/toolsrc/src/vcpkg/base/system.cpp index f49632cac085f8..60787727ef59f3 100644 --- a/toolsrc/src/vcpkg/base/system.cpp +++ b/toolsrc/src/vcpkg/base/system.cpp @@ -105,6 +105,19 @@ namespace vcpkg #endif // defined(_WIN32) } + ExpectedS System::get_home_dir() noexcept + { +#ifdef _WIN32 + auto maybe_home = System::get_environment_variable("USERPROFILE"); + if (!maybe_home.has_value() || maybe_home.get()->empty()) + return {"unable to read %USERPROFILE%", ExpectedRightTag{}}; +#else + auto maybe_home = System::get_environment_variable("HOME"); + if (!maybe_home.has_value() || maybe_home.get()->empty()) return {"unable to read $HOME", ExpectedRightTag{}}; +#endif + return {std::move(*maybe_home.get()), ExpectedLeftTag{}}; + } + #if defined(_WIN32) static bool is_string_keytype(const DWORD hkey_type) { diff --git a/toolsrc/src/vcpkg/base/system.process.cpp b/toolsrc/src/vcpkg/base/system.process.cpp index 2bb61455e8d198..33d44ab6c2b9bc 100644 --- a/toolsrc/src/vcpkg/base/system.process.cpp +++ b/toolsrc/src/vcpkg/base/system.process.cpp @@ -492,7 +492,7 @@ namespace vcpkg std::wstring out_env; - while (1) + for (;;) { auto eq = std::find(it, e, '='); if (eq == e) break; diff --git a/toolsrc/src/vcpkg/binarycaching.cpp b/toolsrc/src/vcpkg/binarycaching.cpp index c8e77d938f58e6..87738c039c42d6 100644 --- a/toolsrc/src/vcpkg/binarycaching.cpp +++ b/toolsrc/src/vcpkg/binarycaching.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -60,6 +61,10 @@ namespace struct ArchivesBinaryProvider : IBinaryProvider { + ArchivesBinaryProvider(std::vector&& read_dirs, std::vector&& write_dirs) + : m_read_dirs(std::move(read_dirs)), m_write_dirs(std::move(write_dirs)) + { + } ~ArchivesBinaryProvider() = default; void prefetch() override {} RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override @@ -68,121 +73,136 @@ namespace auto& spec = action.spec; auto& fs = paths.get_filesystem(); std::error_code ec; - const fs::path archives_root_dir = paths.root / "archives"; - const std::string archive_name = abi_tag + ".zip"; - const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name; - const fs::path archive_path = archives_root_dir / archive_subpath; - const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; - if (fs.exists(archive_path)) + for (auto&& archives_root_dir : m_read_dirs) { - System::print2("Using cached binary package: ", archive_path.u8string(), "\n"); + const std::string archive_name = abi_tag + ".zip"; + const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name; + const fs::path archive_path = archives_root_dir / archive_subpath; + if (fs.exists(archive_path)) + { + System::print2("Using cached binary package: ", archive_path.u8string(), "\n"); - int archive_result = decompress_archive(paths, spec, archive_path).exit_code; + int archive_result = decompress_archive(paths, spec, archive_path).exit_code; - if (archive_result != 0) - { - System::print2("Failed to decompress archive package\n"); - if (action.build_options.purge_decompress_failure == Build::PurgeDecompressFailure::NO) + if (archive_result == 0) { - return RestoreResult::build_failed; + return RestoreResult::success; } else { - System::print2("Purging bad archive\n"); - fs.remove(archive_path, ec); + System::print2("Failed to decompress archive package\n"); + if (action.build_options.purge_decompress_failure == Build::PurgeDecompressFailure::NO) + { + return RestoreResult::build_failed; + } + else + { + System::print2("Purging bad archive\n"); + fs.remove(archive_path, ec); + } } } - else - { - return RestoreResult::success; - } } - - if (fs.exists(archive_tombstone_path)) + for (auto&& archives_root_dir : m_read_dirs) { - if (action.build_options.fail_on_tombstone == Build::FailOnTombstone::YES) + const std::string archive_name = abi_tag + ".zip"; + const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name; + const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; + if (fs.exists(archive_tombstone_path)) { - System::print2("Found failure tombstone: ", archive_tombstone_path.u8string(), "\n"); - return RestoreResult::build_failed; + if (action.build_options.fail_on_tombstone == Build::FailOnTombstone::YES) + { + System::print2("Found failure tombstone: ", archive_tombstone_path.u8string(), "\n"); + return RestoreResult::build_failed; + } + else + { + System::print2(System::Color::warning, + "Found failure tombstone: ", + archive_tombstone_path.u8string(), + "\n"); + } } else { - System::print2( - System::Color::warning, "Found failure tombstone: ", archive_tombstone_path.u8string(), "\n"); + const fs::path archive_path = archives_root_dir / archive_subpath; + System::printf("Could not locate cached archive: %s\n", archive_path.u8string()); } } - else - { - System::printf("Could not locate cached archive: %s\n", archive_path.u8string()); - } - return RestoreResult::missing; } void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override { + if (m_write_dirs.empty()) return; const auto& abi_tag = action.package_abi.value_or_exit(VCPKG_LINE_INFO); auto& spec = action.spec; auto& fs = paths.get_filesystem(); - const fs::path archives_root_dir = paths.root / "archives"; - const std::string archive_name = abi_tag + ".zip"; - const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name; - const fs::path archive_path = archives_root_dir / archive_subpath; - const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip"); - compress_directory(paths, paths.package_dir(spec), tmp_archive_path); - fs.create_directories(archive_path.parent_path(), ignore_errors); - std::error_code ec; - fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec); - if (ec) + for (auto&& m_directory : m_write_dirs) { - System::printf(System::Color::warning, - "Failed to store binary cache %s: %s\n", - archive_path.u8string(), - ec.message()); + const fs::path& archives_root_dir = m_directory; + const std::string archive_name = abi_tag + ".zip"; + const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name; + const fs::path archive_path = archives_root_dir / archive_subpath; + + fs.create_directories(archive_path.parent_path(), ignore_errors); + std::error_code ec; + if (m_write_dirs.size() > 1) + fs.copy_file(tmp_archive_path, archive_path, fs::copy_options::overwrite_existing, ec); + else + fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec); + if (ec) + { + System::printf(System::Color::warning, + "Failed to store binary cache %s: %s\n", + archive_path.u8string(), + ec.message()); + } + else + System::printf("Stored binary cache: %s\n", archive_path.u8string()); } - else - System::printf("Stored binary cache: %s\n", archive_path.u8string()); + if (m_write_dirs.size() > 1) fs.remove(tmp_archive_path, ignore_errors); } void push_failure(const VcpkgPaths& paths, const std::string& abi_tag, const PackageSpec& spec) override { + if (m_write_dirs.empty()) return; auto& fs = paths.get_filesystem(); std::error_code ec; - const fs::path archives_root_dir = paths.root / "archives"; - const std::string archive_name = abi_tag + ".zip"; - const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name; - const fs::path archive_path = archives_root_dir / archive_subpath; - const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; - const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name(); - const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; - - if (!fs.exists(archive_tombstone_path)) + for (auto&& m_directory : m_write_dirs) { - // Build failed, store all failure logs in the tombstone. - const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs"; - const auto tmp_log_path_destination = tmp_log_path / spec.name(); - const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip"; - fs.create_directories(tmp_log_path_destination, ignore_errors); - - for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name())) + const fs::path& archives_root_dir = m_directory; + const std::string archive_name = abi_tag + ".zip"; + const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name; + const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; + if (!fs.exists(archive_tombstone_path)) { - if (log_file.path().extension() == ".log") + // Build failed, store all failure logs in the tombstone. + const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs"; + const auto tmp_log_path_destination = tmp_log_path / spec.name(); + const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip"; + fs.create_directories(tmp_log_path_destination, ignore_errors); + + for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name())) { - fs.copy_file(log_file.path(), - tmp_log_path_destination / log_file.path().filename(), - fs::copy_options::none, - ec); + if (log_file.path().extension() == ".log") + { + fs.copy_file(log_file.path(), + tmp_log_path_destination / log_file.path().filename(), + fs::copy_options::none, + ec); + } } - } - compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip"); + compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip"); - fs.create_directories(archive_tombstone_path.parent_path(), ignore_errors); - fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec); + fs.create_directories(archive_tombstone_path.parent_path(), ignore_errors); + fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec); - // clean up temporary directory - fs.remove_all(tmp_log_path, VCPKG_LINE_INFO); + // clean up temporary directory + fs.remove_all(tmp_log_path, VCPKG_LINE_INFO); + } } } RestoreResult precheck(const VcpkgPaths& paths, @@ -192,35 +212,205 @@ namespace const auto& abi_tag = action.package_abi.value_or_exit(VCPKG_LINE_INFO); auto& fs = paths.get_filesystem(); std::error_code ec; - const fs::path archives_root_dir = paths.root / "archives"; - const std::string archive_name = abi_tag + ".zip"; - const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name; - const fs::path archive_path = archives_root_dir / archive_subpath; - const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; + for (auto&& archives_root_dir : m_read_dirs) + { + const std::string archive_name = abi_tag + ".zip"; + const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name; + const fs::path archive_path = archives_root_dir / archive_subpath; + + if (fs.exists(archive_path)) + { + return RestoreResult::success; + } + } + for (auto&& archives_root_dir : m_read_dirs) + { + const std::string archive_name = abi_tag + ".zip"; + const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name; + const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; + + if (purge_tombstones) + { + fs.remove(archive_tombstone_path, ec); // Ignore error + } + else if (fs.exists(archive_tombstone_path)) + { + if (action.build_options.fail_on_tombstone == Build::FailOnTombstone::YES) + { + return RestoreResult::build_failed; + } + } + } + return RestoreResult::missing; + } + + std::vector m_read_dirs, m_write_dirs; + }; +} + +ExpectedS> vcpkg::create_binary_provider_from_configs(const VcpkgPaths& paths, + View args) +{ + std::string env_string = System::get_environment_variable("VCPKG_BINARY_SOURCES").value_or(""); + + // Preserve existing behavior until CI can be updated + // TODO: remove + if (args.size() == 0 && env_string.empty()) + { + auto p = paths.root / fs::u8path("archives"); + return {std::make_unique(std::vector{p}, std::vector{p})}; + } - if (fs.exists(archive_path)) + return create_binary_provider_from_configs_pure(env_string, args); +} +ExpectedS> vcpkg::create_binary_provider_from_configs_pure( + const std::string& env_string, View args) +{ + struct BinaryConfigParser : Parse::ParserBase + { + using Parse::ParserBase::ParserBase; + + std::vector archives_to_read; + std::vector archives_to_write; + + void parse() + { + while (!at_eof()) { - return RestoreResult::success; + std::vector> segments; + + for (;;) + { + SourceLoc loc = cur_loc(); + std::string segment; + for (;;) + { + auto n = match_until([](char32_t ch) { return ch == ',' || ch == '`' || ch == ';'; }); + Strings::append(segment, n); + auto ch = cur(); + if (ch == '\0' || ch == ',' || ch == ';') + break; + else if (ch == '`') + { + ch = next(); + if (ch == '\0') + add_error("unexpected eof: trailing unescaped backticks (`) are not allowed"); + else + Unicode::utf8_append_code_point(segment, ch); + next(); + } + else + Checks::unreachable(VCPKG_LINE_INFO); + } + segments.emplace_back(std::move(loc), std::move(segment)); + + auto ch = cur(); + if (ch == '\0' || ch == ';') + break; + else if (ch == ',') + { + next(); + continue; + } + else + Checks::unreachable(VCPKG_LINE_INFO); + } + + if (segments.size() != 1 || !segments[0].second.empty()) handle_segments(std::move(segments)); + segments.clear(); + if (get_error()) return; + if (cur() == ';') next(); } + } - if (purge_tombstones) + void handle_segments(std::vector>&& segments) + { + if (segments.empty()) return; + if (segments[0].second == "clear") { - fs.remove(archive_tombstone_path, ec); // Ignore error + if (segments.size() != 1) + return add_error("unexpected arguments: binary config 'clear' does not take arguments", + segments[1].first); + archives_to_read.clear(); + archives_to_write.clear(); } - else if (fs.exists(archive_tombstone_path)) + else if (segments[0].second == "files") { - if (action.build_options.fail_on_tombstone == Build::FailOnTombstone::YES) + if (segments.size() < 2) + return add_error("expected arguments: binary config 'files' requires at least a path argument", + segments[0].first); + + auto p = fs::u8path(segments[1].second); + if (!p.is_absolute()) + return add_error("expected arguments: path arguments for binary config strings must be absolute", + segments[1].first); + + if (segments.size() > 3) { - return RestoreResult::build_failed; + return add_error("unexpected arguments: binary config 'files' does not take more than 2 arguments", + segments[3].first); } + else if (segments.size() == 3) + { + if (segments[2].second != "upload") + { + return add_error("unexpected arguments: binary config 'files' can only accept 'upload' as " + "a second argument", + segments[2].first); + } + else + { + archives_to_write.push_back(p); + } + } + archives_to_read.push_back(std::move(p)); } + else if (segments[0].second == "default") + { + if (segments.size() > 2) + return add_error("unexpected arguments: binary config 'default' does not take more than 1 argument", + segments[0].first); - return RestoreResult::missing; + auto maybe_home = System::get_home_dir(); + if (!maybe_home.has_value()) return add_error(maybe_home.error(), segments[0].first); + + auto p = fs::u8path(maybe_home.value_or_exit(VCPKG_LINE_INFO)) / fs::u8path(".vcpkg/archives"); + if (!p.is_absolute()) + return add_error("default path was not absolute: " + p.u8string(), segments[0].first); + if (segments.size() == 2) + { + if (segments[1].second != "upload") + { + return add_error( + "unexpected arguments: binary config 'default' can only accept 'upload' as an argument", + segments[1].first); + } + else + { + archives_to_write.push_back(p); + } + } + archives_to_read.push_back(std::move(p)); + } + else + { + return add_error("unknown binary provider type: valid providers are 'clear', 'default', and 'files'", + segments[0].first); + } } }; -} -std::unique_ptr vcpkg::create_archives_provider() -{ - return std::make_unique(); + BinaryConfigParser env_parser(env_string, "VCPKG_BINARY_SOURCES"); + env_parser.parse(); + if (auto err = env_parser.get_error()) return err->format(); + for (auto&& arg : args) + { + BinaryConfigParser arg_parser(arg, ""); + arg_parser.parse(); + if (auto err = arg_parser.get_error()) return err->format(); + Util::Vectors::append(&env_parser.archives_to_read, arg_parser.archives_to_read); + Util::Vectors::append(&env_parser.archives_to_write, arg_parser.archives_to_write); + } + return {std::make_unique(std::move(env_parser.archives_to_read), + std::move(env_parser.archives_to_write))}; } diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index b3168075dfaa72..5d40e8c2437483 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -39,6 +39,7 @@ namespace vcpkg::Build::Command void perform_and_exit_ex(const FullPackageSpec& full_spec, const SourceControlFileLocation& scfl, const PathsPortFileProvider& provider, + IBinaryProvider& binaryprovider, const VcpkgPaths& paths) { auto var_provider_storage = CMakeVars::make_triplet_cmake_var_provider(paths); @@ -93,7 +94,7 @@ namespace vcpkg::Build::Command action->build_options = build_package_options; const auto build_timer = Chrono::ElapsedTimer::create_started(); - const auto result = Build::build_package(paths, *action, create_archives_provider().get(), status_db); + const auto result = Build::build_package(paths, *action, binaryprovider, status_db); System::print2("Elapsed time for package ", spec, ": ", build_timer, '\n'); if (result.code == BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES) @@ -135,6 +136,9 @@ namespace vcpkg::Build::Command const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE); std::string first_arg = args.command_arguments.at(0); + auto binaryprovider = + create_binary_provider_from_configs(paths, args.binarysources).value_or_exit(VCPKG_LINE_INFO); + const FullPackageSpec spec = Input::check_and_get_full_package_spec( std::move(first_arg), default_triplet, COMMAND_STRUCTURE.example_text); @@ -146,7 +150,7 @@ namespace vcpkg::Build::Command Checks::check_exit(VCPKG_LINE_INFO, scfl != nullptr, "Error: Couldn't find port '%s'", port_name); - perform_and_exit_ex(spec, *scfl, provider, paths); + perform_and_exit_ex(spec, *scfl, provider, *binaryprovider, paths); } } @@ -820,10 +824,10 @@ namespace vcpkg::Build ExtendedBuildResult build_package(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action, - IBinaryProvider* binaries_provider, + IBinaryProvider& binaries_provider, const StatusParagraphs& status_db) { - auto binary_caching_enabled = binaries_provider && action.build_options.binary_caching == BinaryCaching::YES; + auto binary_caching_enabled = action.build_options.binary_caching == BinaryCaching::YES; auto& fs = paths.get_filesystem(); auto& spec = action.spec; @@ -872,7 +876,7 @@ namespace vcpkg::Build const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; if (binary_caching_enabled) { - auto restore = binaries_provider->try_restore(paths, action); + auto restore = binaries_provider.try_restore(paths, action); if (restore == RestoreResult::build_failed) return BuildResult::BUILD_FAILED; else if (restore == RestoreResult::success) @@ -895,12 +899,12 @@ namespace vcpkg::Build if (binary_caching_enabled && result.code == BuildResult::SUCCEEDED) { - binaries_provider->push_success(paths, action); + binaries_provider.push_success(paths, action); } else if (binary_caching_enabled && (result.code == BuildResult::BUILD_FAILED || result.code == BuildResult::POST_BUILD_CHECKS_FAILED)) { - binaries_provider->push_failure(paths, action.package_abi.value_or_exit(VCPKG_LINE_INFO), spec); + binaries_provider.push_failure(paths, action.package_abi.value_or_exit(VCPKG_LINE_INFO), spec); } return result; diff --git a/toolsrc/src/vcpkg/commands.buildexternal.cpp b/toolsrc/src/vcpkg/commands.buildexternal.cpp index 07ad7b6b173c3a..c9ac0158a43fa3 100644 --- a/toolsrc/src/vcpkg/commands.buildexternal.cpp +++ b/toolsrc/src/vcpkg/commands.buildexternal.cpp @@ -1,5 +1,6 @@ #include "pch.h" +#include #include #include #include @@ -20,6 +21,9 @@ namespace vcpkg::Commands::BuildExternal { const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE); + auto binaryprovider = + create_binary_provider_from_configs(paths, args.binarysources).value_or_exit(VCPKG_LINE_INFO); + const FullPackageSpec spec = Input::check_and_get_full_package_spec( std::string(args.command_arguments.at(0)), default_triplet, COMMAND_STRUCTURE.example_text); Input::check_triplet(spec.package_spec.triplet(), paths); @@ -33,6 +37,7 @@ namespace vcpkg::Commands::BuildExternal Checks::check_exit( VCPKG_LINE_INFO, maybe_scfl.has_value(), "could not load control file for %s", spec.package_spec.name()); - Build::Command::perform_and_exit_ex(spec, maybe_scfl.value_or_exit(VCPKG_LINE_INFO), provider, paths); + Build::Command::perform_and_exit_ex( + spec, maybe_scfl.value_or_exit(VCPKG_LINE_INFO), provider, *binaryprovider, paths); } } diff --git a/toolsrc/src/vcpkg/commands.ci.cpp b/toolsrc/src/vcpkg/commands.ci.cpp index f21366a36e896d..0ea23cd8213803 100644 --- a/toolsrc/src/vcpkg/commands.ci.cpp +++ b/toolsrc/src/vcpkg/commands.ci.cpp @@ -256,7 +256,8 @@ namespace vcpkg::Commands::CI const PortFileProvider::PortFileProvider& provider, const CMakeVars::CMakeVarProvider& var_provider, const std::vector& specs, - const bool purge_tombstones) + const bool purge_tombstones, + IBinaryProvider& binaryprovider) { auto ret = std::make_unique(); @@ -306,8 +307,6 @@ namespace vcpkg::Commands::CI Build::compute_all_abis(paths, action_plan, var_provider, {}); - auto binaryprovider = create_archives_provider(); - std::string stdout_buffer; for (auto&& action : action_plan.install_actions) { @@ -322,7 +321,7 @@ namespace vcpkg::Commands::CI p->build_options = build_options; } - auto precheck_result = binaryprovider->precheck(paths, action, purge_tombstones); + auto precheck_result = binaryprovider.precheck(paths, action, purge_tombstones); bool b_will_build = false; std::string state; @@ -393,6 +392,9 @@ namespace vcpkg::Commands::CI System::print2(System::Color::warning, "Warning: Running ci without binary caching!\n"); } + auto binaryprovider = + create_binary_provider_from_configs(paths, args.binarysources).value_or_exit(VCPKG_LINE_INFO); + const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE); std::set exclusions_set; @@ -458,8 +460,13 @@ namespace vcpkg::Commands::CI return FullPackageSpec{spec, std::move(default_features)}; }); - auto split_specs = find_unknown_ports_for_ci( - paths, exclusions_set, provider, var_provider, all_default_full_specs, purge_tombstones); + auto split_specs = find_unknown_ports_for_ci(paths, + exclusions_set, + provider, + var_provider, + all_default_full_specs, + purge_tombstones, + *binaryprovider); PortFileProvider::MapPortFileProvider new_default_provider(split_specs->default_feature_provider); Dependencies::CreateInstallPlanOptions serialize_options; @@ -503,7 +510,8 @@ namespace vcpkg::Commands::CI else { auto collection_timer = Chrono::ElapsedTimer::create_started(); - auto summary = Install::perform(action_plan, Install::KeepGoing::YES, paths, status_db, var_provider); + auto summary = Install::perform( + action_plan, Install::KeepGoing::YES, paths, status_db, *binaryprovider, var_provider); auto collection_time_elapsed = collection_timer.elapsed(); // Adding results for ports that were built or pulled from an archive diff --git a/toolsrc/src/vcpkg/commands.setinstalled.cpp b/toolsrc/src/vcpkg/commands.setinstalled.cpp index 344892fe83a17c..2ef1b5bd2c6219 100644 --- a/toolsrc/src/vcpkg/commands.setinstalled.cpp +++ b/toolsrc/src/vcpkg/commands.setinstalled.cpp @@ -1,13 +1,14 @@ #include "pch.h" #include +#include #include #include #include #include #include -#include #include +#include #include namespace vcpkg::Commands::SetInstalled @@ -35,6 +36,9 @@ namespace vcpkg::Commands::SetInstalled Input::check_triplet(spec.package_spec.triplet(), paths); } + auto binaryprovider = + create_binary_provider_from_configs(paths, args.binarysources).value_or_exit(VCPKG_LINE_INFO); + const Build::BuildPackageOptions install_plan_options = { Build::UseHeadVersion::NO, Build::AllowDownloads::YES, @@ -47,7 +51,6 @@ namespace vcpkg::Commands::SetInstalled Build::FailOnTombstone::NO, }; - PortFileProvider::PathsPortFileProvider provider(paths, args.overlay_ports.get()); auto cmake_vars = CMakeVars::make_triplet_cmake_var_provider(paths); @@ -66,7 +69,8 @@ namespace vcpkg::Commands::SetInstalled std::set all_abis; - for (const auto& action : action_plan.install_actions) { + for (const auto& action : action_plan.install_actions) + { all_abis.insert(action.package_abi.value_or_exit(VCPKG_LINE_INFO)); } @@ -101,7 +105,8 @@ namespace vcpkg::Commands::SetInstalled Dependencies::print_plan(real_action_plan, true); - const auto summary = Install::perform(real_action_plan, Install::KeepGoing::NO, paths, status_db, *cmake_vars); + const auto summary = + Install::perform(real_action_plan, Install::KeepGoing::NO, paths, status_db, *binaryprovider, *cmake_vars); System::print2("\nTotal elapsed time: ", summary.total_elapsed_time, "\n\n"); diff --git a/toolsrc/src/vcpkg/commands.upgrade.cpp b/toolsrc/src/vcpkg/commands.upgrade.cpp index fbce1a421648d9..112a5b6d63143c 100644 --- a/toolsrc/src/vcpkg/commands.upgrade.cpp +++ b/toolsrc/src/vcpkg/commands.upgrade.cpp @@ -1,5 +1,6 @@ #include "pch.h" +#include #include #include #include @@ -41,6 +42,9 @@ namespace vcpkg::Commands::Upgrade const bool no_dry_run = Util::Sets::contains(options.switches, OPTION_NO_DRY_RUN); const KeepGoing keep_going = to_keep_going(Util::Sets::contains(options.switches, OPTION_KEEP_GOING)); + auto binaryprovider = + create_binary_provider_from_configs(paths, args.binarysources).value_or_exit(VCPKG_LINE_INFO); + StatusParagraphs status_db = database_load_check(paths); // Load ports from ports dirs @@ -184,7 +188,7 @@ namespace vcpkg::Commands::Upgrade var_provider.load_tag_vars(action_plan, provider); const Install::InstallSummary summary = - Install::perform(action_plan, keep_going, paths, status_db, var_provider); + Install::perform(action_plan, keep_going, paths, status_db, *binaryprovider, var_provider); System::print2("\nTotal elapsed time: ", summary.total_elapsed_time, "\n\n"); diff --git a/toolsrc/src/vcpkg/export.prefab.cpp b/toolsrc/src/vcpkg/export.prefab.cpp index 30a3ffa5a116ae..3200159d518890 100644 --- a/toolsrc/src/vcpkg/export.prefab.cpp +++ b/toolsrc/src/vcpkg/export.prefab.cpp @@ -254,8 +254,10 @@ namespace vcpkg::Export::Prefab { auto provider = CMakeVars::make_triplet_cmake_var_provider(paths); - auto build_info = build_info_from_triplet(paths, provider, default_triplet); - Checks::check_exit(VCPKG_LINE_INFO, is_supported(build_info), "Currenty supported on android triplets"); + { + auto build_info = build_info_from_triplet(paths, provider, default_triplet); + Checks::check_exit(VCPKG_LINE_INFO, is_supported(build_info), "Currenty supported on android triplets"); + } std::vector available_triplets = paths.get_available_triplets(); diff --git a/toolsrc/src/vcpkg/install.cpp b/toolsrc/src/vcpkg/install.cpp index 1b9b7db34d6aa7..951e4027869128 100644 --- a/toolsrc/src/vcpkg/install.cpp +++ b/toolsrc/src/vcpkg/install.cpp @@ -300,7 +300,7 @@ namespace vcpkg::Install static ExtendedBuildResult perform_install_plan_action(const VcpkgPaths& paths, InstallPlanAction& action, StatusParagraphs& status_db, - IBinaryProvider* binaries_provider) + IBinaryProvider& binaries_provider) { const InstallPlanType& plan_type = action.plan_type; const std::string display_name = action.spec.to_string(); @@ -425,6 +425,7 @@ namespace vcpkg::Install const KeepGoing keep_going, const VcpkgPaths& paths, StatusParagraphs& status_db, + IBinaryProvider& binaryprovider, const CMakeVars::CMakeVarProvider& var_provider) { std::vector results; @@ -457,16 +458,15 @@ namespace vcpkg::Install for (auto&& action : action_plan.already_installed) { results.emplace_back(action.spec, &action); - results.back().build_result = perform_install_plan_action(paths, action, status_db, nullptr); + results.back().build_result = perform_install_plan_action(paths, action, status_db, binaryprovider); } Build::compute_all_abis(paths, action_plan, var_provider, status_db); - auto binary_provider = create_archives_provider(); for (auto&& action : action_plan.install_actions) { with_tracking(action.spec, [&]() { - auto result = perform_install_plan_action(paths, action, status_db, binary_provider.get()); + auto result = perform_install_plan_action(paths, action, status_db, binaryprovider); if (result.code != BuildResult::SUCCEEDED && keep_going == KeepGoing::NO) { @@ -638,6 +638,9 @@ namespace vcpkg::Install // input sanitization const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE); + auto binaryprovider = + create_binary_provider_from_configs(paths, args.binarysources).value_or_exit(VCPKG_LINE_INFO); + const std::vector specs = Util::fmap(args.command_arguments, [&](auto&& arg) { return Input::check_and_get_full_package_spec( std::string(arg), default_triplet, COMMAND_STRUCTURE.example_text); @@ -731,7 +734,7 @@ namespace vcpkg::Install { const auto vs_prompt_view = to_zstring_view(vs_prompt); System::print2(vcpkg::System::Color::warning, - "warning: vcpkg appears to be in a Visual Studio prompt targeting ", + "warning: vcpkg appears to be in a Visual Studio prompt targeting ", vs_prompt_view, " but is installing packages for ", common_triplet.to_string(), @@ -755,7 +758,8 @@ namespace vcpkg::Install Checks::exit_success(VCPKG_LINE_INFO); } - const InstallSummary summary = perform(action_plan, keep_going, paths, status_db, var_provider); + const InstallSummary summary = + perform(action_plan, keep_going, paths, status_db, *binaryprovider, var_provider); System::print2("\nTotal elapsed time: ", summary.total_elapsed_time, "\n\n"); diff --git a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp index d8deae566a64ef..4c49d110846b4f 100644 --- a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp +++ b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp @@ -79,6 +79,21 @@ namespace vcpkg option_field->emplace_back(std::move(new_value)); } + static void parse_cojoined_multivalue(std::string new_value, + const std::string& option_name, + std::vector& option_field) + { + if (new_value.empty()) + { + System::print2(System::Color::error, "Error: expected value after ", option_name, '\n'); + Metrics::g_metrics.lock()->track_property("error", "error option name"); + Help::print_usage(); + Checks::exit_fail(VCPKG_LINE_INFO); + } + + option_field.emplace_back(std::move(new_value)); + } + VcpkgCmdArguments VcpkgCmdArguments::create_from_command_line(const int argc, const CommandLineCharType* const* const argv) { @@ -176,6 +191,12 @@ namespace vcpkg arg.substr(sizeof("--overlay-triplets=") - 1), "--overlay-triplets", args.overlay_triplets); continue; } + if (Strings::starts_with(arg, "--x-binarysource=")) + { + parse_cojoined_multivalue( + arg.substr(sizeof("--x-binarysource=") - 1), "--x-binarysource", args.binarysources); + continue; + } if (arg == "--debug") { parse_switch(true, "debug", args.debug);