diff --git a/scripts/cmake/vcpkg_download_distfile.cmake b/scripts/cmake/vcpkg_download_distfile.cmake index b2dd9d42c48e4c..0582fc6f2ed83d 100644 --- a/scripts/cmake/vcpkg_download_distfile.cmake +++ b/scripts/cmake/vcpkg_download_distfile.cmake @@ -72,6 +72,11 @@ function(vcpkg_download_distfile VAR) message(FATAL_ERROR "vcpkg_download_distfile must not be passed both SHA512 and SKIP_SHA512.") endif() endif() + + if (DEFINED ENV{VCPKG_EXPERIMENTAL_MIRROR_URL} AND NOT ENV{VCPKG_EXPERIMENTAL_MIRROR_URL} STREQUAL "") + message(STATUS "Use vcpkg mirror $ENV{VCPKG_EXPERIMENTAL_MIRROR_URL}") + set(vcpkg_download_distfile_URLS "ftp://$ENV{VCPKG_EXPERIMENTAL_MIRROR_URL}/${vcpkg_download_distfile_SHA512}") + endif() set(downloaded_file_path ${DOWNLOADS}/${vcpkg_download_distfile_FILENAME}) set(download_file_path_part "${DOWNLOADS}/temp/${vcpkg_download_distfile_FILENAME}") diff --git a/toolsrc/include/pch.h b/toolsrc/include/pch.h index c8ca1ea7c22f5d..602c014ca584fc 100644 --- a/toolsrc/include/pch.h +++ b/toolsrc/include/pch.h @@ -8,7 +8,6 @@ #if defined(_WIN32) #include #include -#include #endif #include diff --git a/toolsrc/include/vcpkg/base/downloads.h b/toolsrc/include/vcpkg/base/downloads.h index 61c79248852130..ee3ee2b518ec05 100644 --- a/toolsrc/include/vcpkg/base/downloads.h +++ b/toolsrc/include/vcpkg/base/downloads.h @@ -4,6 +4,16 @@ namespace vcpkg::Downloads { + void winhttp_download_file(Files::Filesystem& fs, + ZStringView target_file_path, + StringView hostname, + StringView url_path); + + void ftp_download_file(Files::Filesystem& fs, + ZStringView target_file_path, + StringView hostname, + StringView url_path); + void verify_downloaded_file_hash(const Files::Filesystem& fs, const std::string& url, const fs::path& path, diff --git a/toolsrc/include/vcpkg/commands.mirror.h b/toolsrc/include/vcpkg/commands.mirror.h new file mode 100644 index 00000000000000..2ad6b4ec7aa709 --- /dev/null +++ b/toolsrc/include/vcpkg/commands.mirror.h @@ -0,0 +1,15 @@ +#pragma once + +#include + +namespace vcpkg::Commands::Mirror +{ + void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths, Triplet default_triplet); + + struct MirrorCommand : Commands::TripletCommand + { + virtual void perform_and_exit(const VcpkgCmdArguments& inArgs, + const VcpkgPaths& paths, + Triplet default_triplet) const override; + }; +} diff --git a/toolsrc/include/vcpkg/install.h b/toolsrc/include/vcpkg/install.h index f85d48d8d14513..ab0f9bb4ed2721 100644 --- a/toolsrc/include/vcpkg/install.h +++ b/toolsrc/include/vcpkg/install.h @@ -33,6 +33,21 @@ namespace vcpkg::Install const Dependencies::InstallPlanAction* action; }; + struct TrackedPackageInstallGuard + { + SpecSummary* current_summary = nullptr; + Chrono::ElapsedTimer build_timer = Chrono::ElapsedTimer::create_started(); + + TrackedPackageInstallGuard(const size_t package_count, + std::vector& results, + const PackageSpec& spec); + + ~TrackedPackageInstallGuard(); + + TrackedPackageInstallGuard(const TrackedPackageInstallGuard&) = delete; + TrackedPackageInstallGuard& operator=(const TrackedPackageInstallGuard&) = delete; + }; + struct InstallSummary { std::vector results; @@ -101,6 +116,8 @@ namespace vcpkg::Install CMakeUsageInfo get_cmake_usage(const BinaryParagraph& bpgh, const VcpkgPaths& paths); + void print_cmake_information(const BinaryParagraph& bpgh, const VcpkgPaths& paths); + extern const CommandStructure COMMAND_STRUCTURE; void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths, Triplet default_triplet); diff --git a/toolsrc/include/vcpkg/vcpkgcmdarguments.h b/toolsrc/include/vcpkg/vcpkgcmdarguments.h index 626e598be6f58c..56432c882bb70e 100644 --- a/toolsrc/include/vcpkg/vcpkgcmdarguments.h +++ b/toolsrc/include/vcpkg/vcpkgcmdarguments.h @@ -154,6 +154,10 @@ namespace vcpkg constexpr static StringLiteral WAIT_FOR_LOCK_SWITCH = "x-wait-for-lock"; Optional wait_for_lock = nullopt; + constexpr static StringLiteral VCPKG_EXPERIMENTAL_MIRROR_URL = "VCPKG_EXPERIMENTAL_MIRROR_URL"; + std::unique_ptr download_mirror_url; + Optional use_mirror = nullopt; + constexpr static StringLiteral JSON_SWITCH = "x-json"; Optional json = nullopt; diff --git a/toolsrc/include/vcpkg/vcpkgpaths.h b/toolsrc/include/vcpkg/vcpkgpaths.h index 8abc84129426eb..50e08a35031a87 100644 --- a/toolsrc/include/vcpkg/vcpkgpaths.h +++ b/toolsrc/include/vcpkg/vcpkgpaths.h @@ -104,6 +104,8 @@ namespace vcpkg fs::path ports_cmake; + std::string vcpkg_mirror_url; + const fs::path& get_tool_exe(const std::string& tool) const; const std::string& get_tool_version(const std::string& tool) const; diff --git a/toolsrc/src/vcpkg/base/downloads.cpp b/toolsrc/src/vcpkg/base/downloads.cpp index 40f0494f9c2199..24515a1dc021d3 100644 --- a/toolsrc/src/vcpkg/base/downloads.cpp +++ b/toolsrc/src/vcpkg/base/downloads.cpp @@ -10,134 +10,6 @@ namespace vcpkg::Downloads { -#if defined(_WIN32) - static void winhttp_download_file(Files::Filesystem& fs, - ZStringView target_file_path, - StringView hostname, - StringView url_path) - { - // Make sure the directories are present, otherwise fopen_s fails - const auto dir = fs::path(target_file_path.c_str()).parent_path(); - std::error_code ec; - fs.create_directories(dir, ec); - Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not create directories %s", fs::u8string(dir)); - - FILE* f = nullptr; - const errno_t err = fopen_s(&f, target_file_path.c_str(), "wb"); - Checks::check_exit(VCPKG_LINE_INFO, - !err, - "Could not download https://%s%s. Failed to open file %s. Error code was %s", - hostname, - url_path, - target_file_path, - std::to_string(err)); - ASSUME(f != nullptr); - - auto hSession = WinHttpOpen(L"vcpkg/1.0", - IsWindows8Point1OrGreater() ? WINHTTP_ACCESS_TYPE_AUTOMATIC_PROXY - : WINHTTP_ACCESS_TYPE_DEFAULT_PROXY, - WINHTTP_NO_PROXY_NAME, - WINHTTP_NO_PROXY_BYPASS, - 0); - Checks::check_exit(VCPKG_LINE_INFO, hSession, "WinHttpOpen() failed: %d", GetLastError()); - - // If the environment variable HTTPS_PROXY is set - // use that variable as proxy. This situation might exist when user is in a company network - // with restricted network/proxy settings - auto maybe_https_proxy_env = System::get_environment_variable("HTTPS_PROXY"); - if (auto p_https_proxy = maybe_https_proxy_env.get()) - { - std::wstring env_proxy_settings = Strings::to_utf16(*p_https_proxy); - WINHTTP_PROXY_INFO proxy; - proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY; - proxy.lpszProxy = env_proxy_settings.data(); - proxy.lpszProxyBypass = nullptr; - - WinHttpSetOption(hSession, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy)); - } - // Win7 IE Proxy fallback - else if (IsWindows7OrGreater() && !IsWindows8Point1OrGreater()) - { - // First check if any proxy has been found automatically - WINHTTP_PROXY_INFO proxyInfo; - DWORD proxyInfoSize = sizeof(WINHTTP_PROXY_INFO); - auto noProxyFound = !WinHttpQueryOption(hSession, WINHTTP_OPTION_PROXY, &proxyInfo, &proxyInfoSize) || - proxyInfo.dwAccessType == WINHTTP_ACCESS_TYPE_NO_PROXY; - - // If no proxy was found automatically, use IE's proxy settings, if any - if (noProxyFound) - { - WINHTTP_CURRENT_USER_IE_PROXY_CONFIG ieProxy; - if (WinHttpGetIEProxyConfigForCurrentUser(&ieProxy) && ieProxy.lpszProxy != nullptr) - { - WINHTTP_PROXY_INFO proxy; - proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY; - proxy.lpszProxy = ieProxy.lpszProxy; - proxy.lpszProxyBypass = ieProxy.lpszProxyBypass; - WinHttpSetOption(hSession, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy)); - GlobalFree(ieProxy.lpszProxy); - GlobalFree(ieProxy.lpszProxyBypass); - GlobalFree(ieProxy.lpszAutoConfigUrl); - } - } - } - - // Use Windows 10 defaults on Windows 7 - DWORD secure_protocols(WINHTTP_FLAG_SECURE_PROTOCOL_SSL3 | WINHTTP_FLAG_SECURE_PROTOCOL_TLS1 | - WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 | WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2); - WinHttpSetOption(hSession, WINHTTP_OPTION_SECURE_PROTOCOLS, &secure_protocols, sizeof(secure_protocols)); - - // Specify an HTTP server. - auto hConnect = WinHttpConnect(hSession, Strings::to_utf16(hostname).c_str(), INTERNET_DEFAULT_HTTPS_PORT, 0); - Checks::check_exit(VCPKG_LINE_INFO, hConnect, "WinHttpConnect() failed: %d", GetLastError()); - - // Create an HTTP request handle. - auto hRequest = WinHttpOpenRequest(hConnect, - L"GET", - Strings::to_utf16(url_path).c_str(), - nullptr, - WINHTTP_NO_REFERER, - WINHTTP_DEFAULT_ACCEPT_TYPES, - WINHTTP_FLAG_SECURE); - Checks::check_exit(VCPKG_LINE_INFO, hRequest, "WinHttpOpenRequest() failed: %d", GetLastError()); - - // Send a request. - auto bResults = - WinHttpSendRequest(hRequest, WINHTTP_NO_ADDITIONAL_HEADERS, 0, WINHTTP_NO_REQUEST_DATA, 0, 0, 0); - - Checks::check_exit(VCPKG_LINE_INFO, bResults, "WinHttpSendRequest() failed: %d", GetLastError()); - - // End the request. - bResults = WinHttpReceiveResponse(hRequest, NULL); - Checks::check_exit(VCPKG_LINE_INFO, bResults, "WinHttpReceiveResponse() failed: %d", GetLastError()); - - std::vector buf; - - size_t total_downloaded_size = 0; - DWORD dwSize = 0; - do - { - DWORD downloaded_size = 0; - bResults = WinHttpQueryDataAvailable(hRequest, &dwSize); - Checks::check_exit(VCPKG_LINE_INFO, bResults, "WinHttpQueryDataAvailable() failed: %d", GetLastError()); - - if (buf.size() < dwSize) buf.resize(static_cast(dwSize) * 2); - - bResults = WinHttpReadData(hRequest, (LPVOID)buf.data(), dwSize, &downloaded_size); - Checks::check_exit(VCPKG_LINE_INFO, bResults, "WinHttpReadData() failed: %d", GetLastError()); - fwrite(buf.data(), 1, downloaded_size, f); - - total_downloaded_size += downloaded_size; - } while (dwSize > 0); - - WinHttpCloseHandle(hSession); - WinHttpCloseHandle(hConnect); - WinHttpCloseHandle(hRequest); - fflush(f); - fclose(f); - } -#endif - void verify_downloaded_file_hash(const Files::Filesystem& fs, const std::string& url, const fs::path& path, @@ -178,12 +50,15 @@ namespace vcpkg::Downloads fs.remove(download_path, ec); fs.remove(download_path_part_path, ec); #if defined(_WIN32) - auto url_no_proto = url.substr(8); // drop https:// + auto url_no_proto = url.substr(Strings::starts_with(url, "ftp") ? 6 : 8); // drop ftp:// or https:// auto path_begin = Util::find(url_no_proto, '/'); std::string hostname(url_no_proto.begin(), path_begin); std::string path(path_begin, url_no_proto.end()); - winhttp_download_file(fs, download_path_part, hostname, path); + if (Strings::starts_with(url, "ftp")) + ftp_download_file(fs, download_path_part, hostname, path); + else + winhttp_download_file(fs, download_path_part, hostname, path); #else const auto code = System::cmd_execute( Strings::format(R"(curl -L '%s' --create-dirs --output '%s')", url, download_path_part)); diff --git a/toolsrc/src/vcpkg/base/downloads_ftp.cpp b/toolsrc/src/vcpkg/base/downloads_ftp.cpp new file mode 100644 index 00000000000000..864d1fefd2a390 --- /dev/null +++ b/toolsrc/src/vcpkg/base/downloads_ftp.cpp @@ -0,0 +1,49 @@ +#include "pch.h" + +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#include +#pragma comment(lib, "Wininet") +#endif + +namespace vcpkg::Downloads +{ +#if defined(_WIN32) + void ftp_download_file(Files::Filesystem& fs, + ZStringView target_file_path, + StringView hostname, + StringView url_path) + { + // Make sure the directories are present, otherwise fopen_s fails + const auto dir = fs::path(target_file_path.c_str()).parent_path(); + std::error_code ec; + fs.create_directories(dir, ec); + Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not create directories %s", fs::u8string(dir)); + + HINTERNET hConnect; + HINTERNET hFtpSession; + hConnect = InternetOpen(NULL, INTERNET_OPEN_TYPE_DIRECT, NULL, NULL, 0); + Checks::check_exit(VCPKG_LINE_INFO, hConnect, "InternetOpen() failed: %d", GetLastError()); + + hFtpSession = InternetConnect( + hConnect, hostname.to_string().c_str(), INTERNET_DEFAULT_FTP_PORT, "", "", INTERNET_SERVICE_FTP, 0, 0); + Checks::check_exit(VCPKG_LINE_INFO, hFtpSession, "InternetConnect() failed: %d", GetLastError()); + BOOL bSuc = FtpGetFile(hFtpSession, + url_path.to_string().c_str(), + target_file_path.to_string().c_str(), + FALSE, + FTP_TRANSFER_TYPE_BINARY, + 0, + 0); + Checks::check_exit(VCPKG_LINE_INFO, bSuc == TRUE, "InternetConnect() failed: %d", GetLastError()); + InternetCloseHandle(hFtpSession); + InternetCloseHandle(hConnect); + } +#endif +} diff --git a/toolsrc/src/vcpkg/base/downloads_http.cpp b/toolsrc/src/vcpkg/base/downloads_http.cpp new file mode 100644 index 00000000000000..d164de88891fd3 --- /dev/null +++ b/toolsrc/src/vcpkg/base/downloads_http.cpp @@ -0,0 +1,143 @@ +#include "pch.h" + +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#include +#endif + +namespace vcpkg::Downloads +{ +#if defined(_WIN32) + void winhttp_download_file(Files::Filesystem& fs, + ZStringView target_file_path, + StringView hostname, + StringView url_path) + { + // Make sure the directories are present, otherwise fopen_s fails + const auto dir = fs::path(target_file_path.c_str()).parent_path(); + std::error_code ec; + fs.create_directories(dir, ec); + Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not create directories %s", fs::u8string(dir)); + + FILE* f = nullptr; + const errno_t err = fopen_s(&f, target_file_path.c_str(), "wb"); + Checks::check_exit(VCPKG_LINE_INFO, + !err, + "Could not download https://%s%s. Failed to open file %s. Error code was %s", + hostname, + url_path, + target_file_path, + std::to_string(err)); + ASSUME(f != nullptr); + + auto hSession = WinHttpOpen(L"vcpkg/1.0", + IsWindows8Point1OrGreater() ? WINHTTP_ACCESS_TYPE_AUTOMATIC_PROXY + : WINHTTP_ACCESS_TYPE_DEFAULT_PROXY, + WINHTTP_NO_PROXY_NAME, + WINHTTP_NO_PROXY_BYPASS, + 0); + Checks::check_exit(VCPKG_LINE_INFO, hSession, "WinHttpOpen() failed: %d", GetLastError()); + + // If the environment variable HTTPS_PROXY is set + // use that variable as proxy. This situation might exist when user is in a company network + // with restricted network/proxy settings + auto maybe_https_proxy_env = System::get_environment_variable("HTTPS_PROXY"); + if (auto p_https_proxy = maybe_https_proxy_env.get()) + { + std::wstring env_proxy_settings = Strings::to_utf16(*p_https_proxy); + WINHTTP_PROXY_INFO proxy; + proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY; + proxy.lpszProxy = env_proxy_settings.data(); + proxy.lpszProxyBypass = nullptr; + + WinHttpSetOption(hSession, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy)); + } + // Win7 IE Proxy fallback + else if (IsWindows7OrGreater() && !IsWindows8Point1OrGreater()) + { + // First check if any proxy has been found automatically + WINHTTP_PROXY_INFO proxyInfo; + DWORD proxyInfoSize = sizeof(WINHTTP_PROXY_INFO); + auto noProxyFound = !WinHttpQueryOption(hSession, WINHTTP_OPTION_PROXY, &proxyInfo, &proxyInfoSize) || + proxyInfo.dwAccessType == WINHTTP_ACCESS_TYPE_NO_PROXY; + + // If no proxy was found automatically, use IE's proxy settings, if any + if (noProxyFound) + { + WINHTTP_CURRENT_USER_IE_PROXY_CONFIG ieProxy; + if (WinHttpGetIEProxyConfigForCurrentUser(&ieProxy) && ieProxy.lpszProxy != nullptr) + { + WINHTTP_PROXY_INFO proxy; + proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY; + proxy.lpszProxy = ieProxy.lpszProxy; + proxy.lpszProxyBypass = ieProxy.lpszProxyBypass; + WinHttpSetOption(hSession, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy)); + GlobalFree(ieProxy.lpszProxy); + GlobalFree(ieProxy.lpszProxyBypass); + GlobalFree(ieProxy.lpszAutoConfigUrl); + } + } + } + + // Use Windows 10 defaults on Windows 7 + DWORD secure_protocols(WINHTTP_FLAG_SECURE_PROTOCOL_SSL3 | WINHTTP_FLAG_SECURE_PROTOCOL_TLS1 | + WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 | WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2); + WinHttpSetOption(hSession, WINHTTP_OPTION_SECURE_PROTOCOLS, &secure_protocols, sizeof(secure_protocols)); + + // Specify an HTTP server. + auto hConnect = WinHttpConnect(hSession, Strings::to_utf16(hostname).c_str(), INTERNET_DEFAULT_HTTPS_PORT, 0); + Checks::check_exit(VCPKG_LINE_INFO, hConnect, "WinHttpConnect() failed: %d", GetLastError()); + + // Create an HTTP request handle. + auto hRequest = WinHttpOpenRequest(hConnect, + L"GET", + Strings::to_utf16(url_path).c_str(), + nullptr, + WINHTTP_NO_REFERER, + WINHTTP_DEFAULT_ACCEPT_TYPES, + WINHTTP_FLAG_SECURE); + Checks::check_exit(VCPKG_LINE_INFO, hRequest, "WinHttpOpenRequest() failed: %d", GetLastError()); + + // Send a request. + auto bResults = + WinHttpSendRequest(hRequest, WINHTTP_NO_ADDITIONAL_HEADERS, 0, WINHTTP_NO_REQUEST_DATA, 0, 0, 0); + + Checks::check_exit(VCPKG_LINE_INFO, bResults, "WinHttpSendRequest() failed: %d", GetLastError()); + + // End the request. + bResults = WinHttpReceiveResponse(hRequest, NULL); + Checks::check_exit(VCPKG_LINE_INFO, bResults, "WinHttpReceiveResponse() failed: %d", GetLastError()); + + std::vector buf; + + size_t total_downloaded_size = 0; + DWORD dwSize = 0; + do + { + DWORD downloaded_size = 0; + bResults = WinHttpQueryDataAvailable(hRequest, &dwSize); + Checks::check_exit(VCPKG_LINE_INFO, bResults, "WinHttpQueryDataAvailable() failed: %d", GetLastError()); + + if (buf.size() < dwSize) buf.resize(static_cast(dwSize) * 2); + + bResults = WinHttpReadData(hRequest, (LPVOID)buf.data(), dwSize, &downloaded_size); + Checks::check_exit(VCPKG_LINE_INFO, bResults, "WinHttpReadData() failed: %d", GetLastError()); + fwrite(buf.data(), 1, downloaded_size, f); + + total_downloaded_size += downloaded_size; + } while (dwSize > 0); + + WinHttpCloseHandle(hSession); + WinHttpCloseHandle(hConnect); + WinHttpCloseHandle(hRequest); + fflush(f); + fclose(f); + } +#endif +} diff --git a/toolsrc/src/vcpkg/base/system.process.cpp b/toolsrc/src/vcpkg/base/system.process.cpp index a0cdf0365ccdab..358af2667d71a0 100644 --- a/toolsrc/src/vcpkg/base/system.process.cpp +++ b/toolsrc/src/vcpkg/base/system.process.cpp @@ -300,6 +300,8 @@ namespace vcpkg // Environment variables needed for ssh-agent based authentication L"SSH_AUTH_SOCK", L"SSH_AGENT_PID", + // Environment variables to set the vcpkg mirror address + L"VCPKG_EXPERIMENTAL_MIRROR_URL", // Enables find_package(CUDA) and enable_language(CUDA) in CMake L"CUDA_PATH", L"CUDA_PATH_V9_0", diff --git a/toolsrc/src/vcpkg/commands.cpp b/toolsrc/src/vcpkg/commands.cpp index 6e87c17748d7df..53c625a684f22f 100644 --- a/toolsrc/src/vcpkg/commands.cpp +++ b/toolsrc/src/vcpkg/commands.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -110,19 +111,19 @@ namespace vcpkg::Commands static const BuildExternal::BuildExternalCommand build_external{}; static const Export::ExportCommand export_command{}; static const DependInfo::DependInfoCommand depend_info{}; + static const Mirror::MirrorCommand mirror{}; - static std::vector> t = { - {"install", &install}, - {"x-set-installed", &set_installed}, - {"ci", &ci}, - {"remove", &remove}, - {"upgrade", &upgrade}, - {"build", &build}, - {"env", &env}, - {"build-external", &build_external}, - {"export", &export_command}, - {"depend-info", &depend_info}, - }; + static std::vector> t = {{"install", &install}, + {"x-set-installed", &set_installed}, + {"ci", &ci}, + {"remove", &remove}, + {"upgrade", &upgrade}, + {"build", &build}, + {"env", &env}, + {"build-external", &build_external}, + {"export", &export_command}, + {"depend-info", &depend_info}, + {"x-mirror", &mirror}}; return t; } } diff --git a/toolsrc/src/vcpkg/commands.mirror.cpp b/toolsrc/src/vcpkg/commands.mirror.cpp new file mode 100644 index 00000000000000..9a77cb0f5b4454 --- /dev/null +++ b/toolsrc/src/vcpkg/commands.mirror.cpp @@ -0,0 +1,274 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace vcpkg::Commands::Mirror +{ + using namespace vcpkg; + using namespace Dependencies; + + static constexpr StringLiteral OPTION_DRY_RUN = "dry-run"; + static constexpr StringLiteral OPTION_USE_ARIA2 = "x-use-aria2"; + static constexpr StringLiteral OPTION_WRITE_PACKAGES_CONFIG = "x-write-nuget-packages-config"; + + static constexpr std::array INSTALL_SWITCHES = {{ + {OPTION_DRY_RUN, "Do not actually build or install"}, + {OPTION_USE_ARIA2, "Use aria2 to perform download tasks"}, + }}; + static constexpr std::array INSTALL_SETTINGS = {{ + {OPTION_WRITE_PACKAGES_CONFIG, + "Writes out a NuGet packages.config-formatted file for use with external binary caching.\nSee `vcpkg help " + "binarycaching` for more information."}, + }}; + + const CommandStructure COMMAND_STRUCTURE = { + create_example_string("mirror"), + 0, + 0, + {INSTALL_SWITCHES, INSTALL_SETTINGS}, + nullptr, + }; + + /// + /// + /// Run "install" command. + /// + /// + void perform_and_exit(const VcpkgCmdArguments& inArgs, const VcpkgPaths& paths, Triplet default_triplet) + { + // input sanitization + VcpkgCmdArguments args; + memcpy(&args, &inArgs, sizeof(VcpkgCmdArguments)); + const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE); + + if (args.command_arguments.size()) + { + args.command_arguments.pop_back(); + } + + args.command_arguments = vcpkg::Install::get_all_port_names(paths); + + auto binaryprovider = create_binary_provider_from_configs(args.binary_sources).value_or_exit(VCPKG_LINE_INFO); + + const bool dry_run = Util::Sets::contains(options.switches, OPTION_DRY_RUN); + const bool use_aria2 = Util::Sets::contains(options.switches, (OPTION_USE_ARIA2)); + + auto& fs = paths.get_filesystem(); + + Build::DownloadTool download_tool = Build::DownloadTool::BUILT_IN; + if (use_aria2) download_tool = Build::DownloadTool::ARIA2; + + const Build::BuildPackageOptions install_plan_options = { + Util::Enum::to_enum(false), + Util::Enum::to_enum(true), + Util::Enum::to_enum(true), + Util::Enum::to_enum(true), + Util::Enum::to_enum(true), + Util::Enum::to_enum(false), + download_tool, + Build::PurgeDecompressFailure::NO, + Util::Enum::to_enum(false), + }; + + PortFileProvider::PathsPortFileProvider provider(paths, args.overlay_ports); + auto var_provider_storage = CMakeVars::make_triplet_cmake_var_provider(paths); + auto& var_provider = *var_provider_storage; + + if (auto manifest = paths.get_manifest().get()) + { + Optional pkgsconfig; + auto it_pkgsconfig = options.settings.find(OPTION_WRITE_PACKAGES_CONFIG); + if (it_pkgsconfig != options.settings.end()) + { + pkgsconfig = fs::u8path(it_pkgsconfig->second); + } + auto manifest_path = paths.get_manifest_path().value_or_exit(VCPKG_LINE_INFO); + auto maybe_manifest_scf = SourceControlFile::parse_manifest_file(manifest_path, *manifest); + if (!maybe_manifest_scf) + { + print_error_message(maybe_manifest_scf.error()); + System::print2( + "See https://github.com/Microsoft/vcpkg/tree/master/docs/specifications/manifests.md for " + "more information.\n"); + Checks::exit_fail(VCPKG_LINE_INFO); + } + auto& manifest_scf = *maybe_manifest_scf.value_or_exit(VCPKG_LINE_INFO); + + std::vector features; + auto core_it = Util::find(features, "core"); + features.erase(core_it); + auto specs = resolve_deps_as_top_level(manifest_scf, default_triplet, features, var_provider); + + auto install_plan = Dependencies::create_feature_install_plan(provider, var_provider, specs, {}); + + for (InstallPlanAction& action : install_plan.install_actions) + { + action.build_options = install_plan_options; + action.build_options.use_head_version = Build::UseHeadVersion::NO; + action.build_options.editable = Build::Editable::NO; + } + + Commands::SetInstalled::perform_and_exit_ex(args, + paths, + provider, + *binaryprovider, + var_provider, + std::move(install_plan), + dry_run ? Commands::DryRun::Yes : Commands::DryRun::No, + pkgsconfig); + } + + const std::vector specs = Util::fmap(args.command_arguments, [&](auto&& arg) { + return Input::check_and_get_full_package_spec( + std::string(arg), default_triplet, COMMAND_STRUCTURE.example_text); + }); + + for (auto&& spec : specs) + { + Input::check_triplet(spec.package_spec.triplet(), paths); + } + + // create the plan + System::print2("Computing installation plan...\n"); + StatusParagraphs status_db = database_load_check(paths); + + // Note: action_plan will hold raw pointers to SourceControlFileLocations from this map + auto action_plan = Dependencies::create_feature_install_plan(provider, var_provider, specs, status_db); + + for (auto&& action : action_plan.install_actions) + { + action.build_options = install_plan_options; + if (action.request_type != RequestType::USER_REQUESTED) + { + action.build_options.use_head_version = Build::UseHeadVersion::NO; + action.build_options.editable = Build::Editable::NO; + } + } + + var_provider.load_tag_vars(action_plan, provider); + + // install plan will be empty if it is already installed - need to change this at status paragraph part + Checks::check_exit(VCPKG_LINE_INFO, !action_plan.empty(), "Install plan cannot be empty"); + + // log the plan + std::string specs_string; + for (auto&& remove_action : action_plan.remove_actions) + { + if (!specs_string.empty()) specs_string.push_back(','); + specs_string += "R$" + Hash::get_string_hash(remove_action.spec.to_string(), Hash::Algorithm::Sha256); + } + + for (auto&& install_action : action_plan.install_actions) + { + if (!specs_string.empty()) specs_string.push_back(','); + specs_string += Hash::get_string_hash(install_action.spec.to_string(), Hash::Algorithm::Sha256); + } + +#if defined(_WIN32) + const auto maybe_common_triplet = common_projection( + action_plan.install_actions, [](const InstallPlanAction& to_install) { return to_install.spec.triplet(); }); + if (maybe_common_triplet) + { + const auto& common_triplet = maybe_common_triplet.value_or_exit(VCPKG_LINE_INFO); + const auto maybe_common_arch = common_triplet.guess_architecture(); + if (maybe_common_arch) + { + const auto maybe_vs_prompt = System::guess_visual_studio_prompt_target_architecture(); + if (maybe_vs_prompt) + { + const auto common_arch = maybe_common_arch.value_or_exit(VCPKG_LINE_INFO); + const auto vs_prompt = maybe_vs_prompt.value_or_exit(VCPKG_LINE_INFO); + if (common_arch != vs_prompt) + { + const auto vs_prompt_view = to_zstring_view(vs_prompt); + System::print2(vcpkg::System::Color::warning, + "warning: vcpkg appears to be in a Visual Studio prompt targeting ", + vs_prompt_view, + " but is installing packages for ", + common_triplet.to_string(), + ". Consider using --triplet ", + vs_prompt_view, + "-windows or --triplet ", + vs_prompt_view, + "-uwp.\n"); + } + } + } + } +#endif // defined(_WIN32) + + Metrics::g_metrics.lock()->track_property("installplan_1", specs_string); + + Dependencies::print_plan(action_plan, true, paths.ports); + + auto it_pkgsconfig = options.settings.find(OPTION_WRITE_PACKAGES_CONFIG); + if (it_pkgsconfig != options.settings.end()) + { + Build::compute_all_abis(paths, action_plan, var_provider, status_db); + + auto pkgsconfig_path = Files::combine(paths.original_cwd, fs::u8path(it_pkgsconfig->second)); + auto pkgsconfig_contents = generate_nuget_packages_config(action_plan); + fs.write_contents(pkgsconfig_path, pkgsconfig_contents, VCPKG_LINE_INFO); + System::print2("Wrote NuGet packages config information to ", fs::u8string(pkgsconfig_path), "\n"); + } + + if (dry_run) + { + Checks::exit_success(VCPKG_LINE_INFO); + } + + const vcpkg::Install::InstallSummary summary = vcpkg::Install::perform(action_plan, + vcpkg::Install::KeepGoing::YES, + paths, + status_db, + null_binary_provider(), + Build::null_build_logs_recorder(), + var_provider); + + System::print2("\nTotal elapsed time: ", summary.total_elapsed_time, "\n\n"); + + summary.print(); + + for (auto&& result : summary.results) + { + if (!result.action) continue; + if (result.action->request_type != RequestType::USER_REQUESTED) continue; + auto bpgh = result.get_binary_paragraph(); + if (!bpgh) continue; + vcpkg::Install::print_cmake_information(*bpgh, paths); + } + + // rename the downloaded files + std::error_code ec; + auto downloads = fs.get_files_non_recursive(paths.downloads); + + for (auto&& file : downloads) + { + if (!fs.is_directory(file)) + { + std::string fileHash = + vcpkg::Hash::get_file_hash(VCPKG_LINE_INFO, fs, file, vcpkg::Hash::Algorithm::Sha512); + auto newpath = file.parent_path(); + newpath += "\\"; + newpath += fileHash; + fs.rename(file, newpath, ec); + } + } + + Checks::exit_success(VCPKG_LINE_INFO); + } + + void MirrorCommand::perform_and_exit(const VcpkgCmdArguments& args, + const VcpkgPaths& paths, + Triplet default_triplet) const + { + Mirror::perform_and_exit(args, paths, default_triplet); + } +} diff --git a/toolsrc/src/vcpkg/install.cpp b/toolsrc/src/vcpkg/install.cpp index 99d17f45939ad2..5326fb2734856c 100644 --- a/toolsrc/src/vcpkg/install.cpp +++ b/toolsrc/src/vcpkg/install.cpp @@ -429,30 +429,20 @@ namespace vcpkg::Install } } - struct TrackedPackageInstallGuard + TrackedPackageInstallGuard::TrackedPackageInstallGuard(const size_t package_count, + std::vector& results, + const PackageSpec& spec) { - SpecSummary* current_summary = nullptr; - Chrono::ElapsedTimer build_timer = Chrono::ElapsedTimer::create_started(); - - TrackedPackageInstallGuard(const size_t package_count, - std::vector& results, - const PackageSpec& spec) - { - results.emplace_back(spec, nullptr); - current_summary = &results.back(); - System::printf("Starting package %zd/%zd: %s\n", results.size(), package_count, spec.to_string()); - } - - ~TrackedPackageInstallGuard() - { - current_summary->timing = build_timer.elapsed(); - System::printf( - "Elapsed time for package %s: %s\n", current_summary->spec.to_string(), current_summary->timing); - } + results.emplace_back(spec, nullptr); + current_summary = &results.back(); + System::printf("Starting package %zd/%zd: %s\n", results.size(), package_count, spec.to_string()); + } - TrackedPackageInstallGuard(const TrackedPackageInstallGuard&) = delete; - TrackedPackageInstallGuard& operator=(const TrackedPackageInstallGuard&) = delete; - }; + TrackedPackageInstallGuard::~TrackedPackageInstallGuard() + { + current_summary->timing = build_timer.elapsed(); + System::printf("Elapsed time for package %s: %s\n", current_summary->spec.to_string(), current_summary->timing); + } InstallSummary perform(ActionPlan& action_plan, const KeepGoing keep_going, @@ -572,7 +562,7 @@ namespace vcpkg::Install nullptr, }; - static void print_cmake_information(const BinaryParagraph& bpgh, const VcpkgPaths& paths) + void print_cmake_information(const BinaryParagraph& bpgh, const VcpkgPaths& paths) { auto usage = get_cmake_usage(bpgh, paths); diff --git a/toolsrc/src/vcpkg/metrics.cpp b/toolsrc/src/vcpkg/metrics.cpp index 241531df38a24f..373bf2ede2a093 100644 --- a/toolsrc/src/vcpkg/metrics.cpp +++ b/toolsrc/src/vcpkg/metrics.cpp @@ -10,6 +10,8 @@ #include #if defined(_WIN32) +#include + #pragma comment(lib, "version") #pragma comment(lib, "winhttp") #endif diff --git a/toolsrc/src/vcpkg/tools.cpp b/toolsrc/src/vcpkg/tools.cpp index 5531a1daca3237..16f5e744eefebc 100644 --- a/toolsrc/src/vcpkg/tools.cpp +++ b/toolsrc/src/vcpkg/tools.cpp @@ -10,6 +10,7 @@ #include #include +#include #include namespace vcpkg @@ -95,9 +96,22 @@ namespace vcpkg StringView::find_exactly_one_enclosed(tool_data, "", "").to_string(); const std::string exe_relative_path = StringView::find_exactly_one_enclosed(tool_data, "", "").to_string(); - const std::string url = StringView::find_exactly_one_enclosed(tool_data, "", "").to_string(); const std::string sha512 = StringView::find_exactly_one_enclosed(tool_data, "", "").to_string(); + + std::string url; + + if (!paths.vcpkg_mirror_url.empty()) + { + url = "ftp://"; + url += paths.vcpkg_mirror_url; + url += "//" + sha512; + } + else + { + url = StringView::find_exactly_one_enclosed(tool_data, "", "").to_string(); + } + auto archive_name = StringView::find_at_most_one_enclosed(tool_data, "", ""); const Optional> version = parse_version_string(version_as_string); diff --git a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp index d56de23a88a959..2e23d92e5e4193 100644 --- a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp +++ b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp @@ -309,6 +309,7 @@ namespace vcpkg {FEATURE_PACKAGES_SWITCH, &VcpkgCmdArguments::feature_packages}, {BINARY_CACHING_SWITCH, &VcpkgCmdArguments::binary_caching}, {WAIT_FOR_LOCK_SWITCH, &VcpkgCmdArguments::wait_for_lock}, + {VCPKG_EXPERIMENTAL_MIRROR_URL, &VcpkgCmdArguments::use_mirror}, {JSON_SWITCH, &VcpkgCmdArguments::json}, }; @@ -679,6 +680,20 @@ namespace vcpkg } } + if (!download_mirror_url) + { + const auto vcpkg_mirror_env = vcpkg::System::get_environment_variable(VCPKG_EXPERIMENTAL_MIRROR_URL); + if (const auto unpacked = vcpkg_mirror_env.get()) + { + download_mirror_url = std::make_unique(*unpacked); + use_mirror = true; + } + else + { + use_mirror = false; + } + } + const auto vcpkg_feature_flags_env = System::get_environment_variable(FEATURE_FLAGS_ENV); if (const auto v = vcpkg_feature_flags_env.get()) { diff --git a/toolsrc/src/vcpkg/vcpkgpaths.cpp b/toolsrc/src/vcpkg/vcpkgpaths.cpp index 9f4ffce6b0d1bf..33174ef33dda80 100644 --- a/toolsrc/src/vcpkg/vcpkgpaths.cpp +++ b/toolsrc/src/vcpkg/vcpkgpaths.cpp @@ -352,6 +352,8 @@ If you wish to silence this error and use classic mode, you can: ports_cmake = filesystem.canonical(VCPKG_LINE_INFO, scripts / fs::u8path("ports.cmake")); + if (args.download_mirror_url) vcpkg_mirror_url = *args.download_mirror_url.get(); + for (auto&& overlay_triplets_dir : args.overlay_triplets) { m_pimpl->triplets_dirs.emplace_back(