From 6ff2bfdc557da135a9629933505725b2a409fecf Mon Sep 17 00:00:00 2001 From: Leander Schulten Date: Sat, 21 Aug 2021 03:00:05 +0200 Subject: [PATCH 1/4] x-download: Add option --only-warn-sha512-mismatch that cause the only a warning instead of an error is emitted when there is a sha512 mismatch --- include/vcpkg/base/downloads.h | 16 ++++++++++---- src/vcpkg/base/downloads.cpp | 38 +++++++++++++++++++++++--------- src/vcpkg/commands.xdownload.cpp | 8 ++++++- 3 files changed, 47 insertions(+), 15 deletions(-) diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h index c89d5370bd..84753400ef 100644 --- a/include/vcpkg/base/downloads.h +++ b/include/vcpkg/base/downloads.h @@ -45,6 +45,11 @@ namespace vcpkg::Downloads bool m_block_origin = false; }; + enum class Sha512MismatchAction : bool + { + Warn, + Error, + }; // Handles downloading and uploading to a content addressable mirror struct DownloadManager { @@ -55,23 +60,26 @@ namespace vcpkg::Downloads void download_file(Filesystem& fs, const std::string& url, const Path& download_path, - const Optional& sha512) const + const Optional& sha512, + Sha512MismatchAction mismatch_action = Sha512MismatchAction::Error) const { - this->download_file(fs, url, {}, download_path, sha512); + this->download_file(fs, url, {}, download_path, sha512, mismatch_action); } void download_file(Filesystem& fs, const std::string& url, View headers, const Path& download_path, - const Optional& sha512) const; + const Optional& sha512, + Sha512MismatchAction mismatch_action = Sha512MismatchAction::Error) const; // Returns url that was successfully downloaded from std::string download_file(Filesystem& fs, View urls, View headers, const Path& download_path, - const Optional& sha512) const; + const Optional& sha512, + Sha512MismatchAction mismatch_action = Sha512MismatchAction::Error) const; ExpectedS put_file_to_mirror(const Filesystem& fs, const Path& file_to_put, StringView sha512) const; diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 0bcdab3aa6..17fe7156c8 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -239,6 +239,7 @@ namespace vcpkg::Downloads static bool check_downloaded_file_hash(Filesystem& fs, const Optional& hash, + Sha512MismatchAction mismatch_action, StringView sanitized_url, const Path& download_part_path, std::string& errors) @@ -248,6 +249,11 @@ namespace vcpkg::Downloads auto maybe_error = try_verify_downloaded_file_hash(fs, sanitized_url, download_part_path, *p); if (auto err = maybe_error.get()) { + if (mismatch_action == Sha512MismatchAction::Warn) + { + print2(Color::warning, *err, '\n'); + return true; + } Strings::append(errors, *err, '\n'); return false; } @@ -481,6 +487,7 @@ namespace vcpkg::Downloads View headers, const Path& download_path, const Optional& sha512, + Sha512MismatchAction mismatch_action, const std::vector& secrets, std::string& errors) { @@ -505,7 +512,8 @@ namespace vcpkg::Downloads { if (download_winhttp(fs, download_path_part_path, split_uri, url, secrets, errors)) { - if (check_downloaded_file_hash(fs, sha512, url, download_path_part_path, errors)) + if (check_downloaded_file_hash( + fs, sha512, only_warn_sha512_mismatch, url, download_path_part_path, errors)) { fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); return true; @@ -536,7 +544,7 @@ namespace vcpkg::Downloads return false; } - if (check_downloaded_file_hash(fs, sha512, sanitized_url, download_path_part_path, errors)) + if (check_downloaded_file_hash(fs, sha512, mismatch_action, sanitized_url, download_path_part_path, errors)) { fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); return true; @@ -549,12 +557,14 @@ namespace vcpkg::Downloads View headers, const Path& download_path, const Optional& sha512, + Sha512MismatchAction mismatch_action, const std::vector& secrets, std::string& errors) { for (auto&& url : urls) { - if (try_download_file(fs, url, headers, download_path, sha512, secrets, errors)) return url; + if (try_download_file(fs, url, headers, download_path, sha512, mismatch_action, secrets, errors)) + return url; } return nullopt; } @@ -569,16 +579,18 @@ namespace vcpkg::Downloads const std::string& url, View headers, const Path& download_path, - const Optional& sha512) const + const Optional& sha512, + Sha512MismatchAction mismatch_action) const { - this->download_file(fs, View(&url, 1), headers, download_path, sha512); + this->download_file(fs, View(&url, 1), headers, download_path, sha512, mismatch_action); } std::string DownloadManager::download_file(Filesystem& fs, View urls, View headers, const Path& download_path, - const Optional& sha512) const + const Optional& sha512, + Sha512MismatchAction mismatch_action) const { std::string errors; if (auto hash = sha512.get()) @@ -586,8 +598,14 @@ namespace vcpkg::Downloads if (auto read_template = m_config.m_read_url_template.get()) { auto read_url = Strings::replace_all(*read_template, "", *hash); - if (Downloads::try_download_file( - fs, read_url, m_config.m_read_headers, download_path, sha512, m_config.m_secrets, errors)) + if (Downloads::try_download_file(fs, + read_url, + m_config.m_read_headers, + download_path, + sha512, + mismatch_action, + m_config.m_secrets, + errors)) return read_url; } } @@ -607,8 +625,8 @@ namespace vcpkg::Downloads } else { - auto maybe_url = - try_download_files(fs, urls, headers, download_path, sha512, m_config.m_secrets, errors); + auto maybe_url = try_download_files( + fs, urls, headers, download_path, sha512, mismatch_action, m_config.m_secrets, errors); if (auto url = maybe_url.get()) { if (auto hash = sha512.get()) diff --git a/src/vcpkg/commands.xdownload.cpp b/src/vcpkg/commands.xdownload.cpp index 3ecc6827d0..dd733270e9 100644 --- a/src/vcpkg/commands.xdownload.cpp +++ b/src/vcpkg/commands.xdownload.cpp @@ -14,12 +14,15 @@ namespace vcpkg::Commands::X_Download static constexpr StringLiteral OPTION_STORE = "store"; static constexpr StringLiteral OPTION_SKIP_SHA512 = "skip-sha512"; static constexpr StringLiteral OPTION_SHA512 = "sha512"; + static constexpr StringLiteral OPTION_ONLY_WARN_FOR_SHA512_MISMATCH = "only-warn-sha512-mismatch"; static constexpr StringLiteral OPTION_URL = "url"; static constexpr StringLiteral OPTION_HEADER = "header"; static constexpr CommandSwitch FETCH_SWITCHES[] = { {OPTION_STORE, "Indicates the file should be stored instead of fetched"}, {OPTION_SKIP_SHA512, "Do not check the SHA512 of the downloaded file"}, + {OPTION_ONLY_WARN_FOR_SHA512_MISMATCH, + "Only warn if the SHA512 of the downloaded file mismatched instead of fail"}, }; static constexpr CommandSetting FETCH_SETTINGS[] = { {OPTION_SHA512, "The hash of the file to be downloaded"}, @@ -99,6 +102,9 @@ namespace vcpkg::Commands::X_Download auto file = fs.absolute(args.command_arguments[0], VCPKG_LINE_INFO); auto sha = get_sha512_check(args, parsed); + using Action = Downloads::Sha512MismatchAction; + auto mismatch_action = + Util::Sets::contains(parsed.switches, OPTION_ONLY_WARN_FOR_SHA512_MISMATCH) ? Action::Warn : Action::Error; // Is this a store command? if (Util::Sets::contains(parsed.switches, OPTION_STORE)) @@ -139,7 +145,7 @@ namespace vcpkg::Commands::X_Download urls = it_urls->second; } - download_manager.download_file(fs, urls, headers, file, sha); + download_manager.download_file(fs, urls, headers, file, sha, mismatch_action); Checks::exit_success(VCPKG_LINE_INFO); } } From bf26848e5d90ad5341e17369fb6eac2beb5339cd Mon Sep 17 00:00:00 2001 From: Leander Schulten Date: Sat, 21 Aug 2021 03:01:47 +0200 Subject: [PATCH 2/4] install: add option x-auto-update-sha512-hashes that updates wrong sha512 hashes through new ones --- include/vcpkg/build.h | 7 ++ src/vcpkg/build.cpp | 190 ++++++++++++++++++++++++++++++++++++++++++ src/vcpkg/install.cpp | 5 ++ 3 files changed, 202 insertions(+) diff --git a/include/vcpkg/build.h b/include/vcpkg/build.h index b91bee5940..ef48a036ec 100644 --- a/include/vcpkg/build.h +++ b/include/vcpkg/build.h @@ -152,6 +152,12 @@ namespace vcpkg::Build YES }; + enum class AutoUpdateMismatchedSHA512 + { + NO = 0, + YES + }; + struct BuildPackageOptions { BuildMissing build_missing; @@ -165,6 +171,7 @@ namespace vcpkg::Build PurgeDecompressFailure purge_decompress_failure; Editable editable; BackcompatFeatures backcompat_features; + AutoUpdateMismatchedSHA512 auto_update_mismatched_sha512; }; static constexpr BuildPackageOptions default_build_package_options{ diff --git a/src/vcpkg/build.cpp b/src/vcpkg/build.cpp index 510473a670..1e72c81e4e 100644 --- a/src/vcpkg/build.cpp +++ b/src/vcpkg/build.cpp @@ -750,6 +750,8 @@ namespace vcpkg::Build {"_VCPKG_DOWNLOAD_TOOL", to_string(action.build_options.download_tool)}, {"_VCPKG_EDITABLE", Util::Enum::to_bool(action.build_options.editable) ? "1" : "0"}, {"_VCPKG_NO_DOWNLOADS", !Util::Enum::to_bool(action.build_options.allow_downloads) ? "1" : "0"}, + {"Z_VCPKG_SHA512_MISMATCH_NO_ERROR", + Util::Enum::to_bool(action.build_options.auto_update_mismatched_sha512) ? "1" : "0"}, }; for (auto cmake_arg : args.cmake_args) @@ -849,6 +851,189 @@ namespace vcpkg::Build } } + struct Sha512Scanner + { + private: + struct Entry + { + std::string old_hash; + std::string new_hash; + struct StackTraceEntry + { + std::string path; + int line; + }; + std::string complete_stack_trace; + std::vector stack_trace; + }; + std::vector entries; + Entry current_entry; + enum class State + { + None, + Start, + CallStack + } state = State::None; + + public: + void parse_input(StringView input) + { + if (input == "\n") + { + parse_line(""); + } + else + { + for (const auto line : Strings::split(input, '\n')) + { + parse_line(line); + } + } + } + + private: + void parse_line(const std::string& line) + { + auto index = std::string::npos; + if (Strings::starts_with(line, "File does not have the expected hash")) + { + state = State::Start; + } + else if (state == State::None) + { + return; // don't check every line for performance reasons + } + else if ((index = line.find("Expected hash : [ ")) != std::string::npos) + { + index += 18; // length of "Expected hash : [ " + if (line.length() < index + 128) + { + state = State::None; + return; + } + current_entry.old_hash = line.substr(index, 128); + } + else if ((index = line.find("Actual hash : [ ")) != std::string::npos) + { + index += 16; // length of "Actual hash : [ " + if (line.length() < index + 128) + { + state = State::None; + return; + } + current_entry.new_hash = line.substr(index, 128); + } + else if (line.find("Call Stack (most recent call first):") != std::string::npos) + { + state = State::CallStack; + } + else if (state == State::CallStack) + { + if (line.empty()) + { + state = State::None; + entries.push_back(current_entry); + current_entry = Entry(); + return; + } + current_entry.complete_stack_trace += line; + current_entry.complete_stack_trace += '\n'; + if (line.length() < 5) // a real entry can not only constis out of 5 chars + return; + parse_stack_trace_entry(line.substr(2)); + } + } + + void parse_stack_trace_entry(std::string line) + { + auto colon = line.find(":"); + const auto printError = [&] { + print2(Color::error, + "The Strack trace line should have the format `path/fileName.cmake: " + "(functionName)` but is ", + line); + }; + if (colon == std::string::npos) + { + printError(); + return; + } + auto space = line.find(" ", colon); + if (space == std::string::npos) + { + printError(); + return; + } + const auto path = line.substr(0, colon); + if (Strings::starts_with(path, "scripts/")) + { + return; + } + try + { + auto line_number = std::stoi(line.substr(colon + 1, space)); + current_entry.stack_trace.push_back(Entry::StackTraceEntry{path, line_number}); + } + catch (std::exception&) + { + printError(); + return; + } + } + + public: + void replace_in_files(const VcpkgPaths& paths) + { + auto& fs = paths.get_filesystem(); + for (const auto& entry : entries) + { + for (const auto& stackTraceEntry : entry.stack_trace) + { + const auto file_path = paths.root / stackTraceEntry.path; + auto lines = fs.read_lines(file_path, VCPKG_LINE_INFO); + for (int i = stackTraceEntry.line - 1; + i < std::min(stackTraceEntry.line + 10, static_cast(lines.size())); + ++i) + { + auto& line = lines[i]; + const auto sha512_index = line.find("SHA512"); + if (sha512_index == std::string::npos) continue; + const auto sha_start = line.find_first_not_of(" ", sha512_index + 6); + if (sha_start == std::string::npos) continue; + auto sha_end = line.find_first_not_of("0123456789abcdef", sha_start); + if (sha_end == std::string::npos) sha_end = line.size(); + const auto sha = line.substr(sha_start, sha_end); + if (sha == "0" || sha == entry.old_hash) + { + line.replace(sha_start, sha_end - sha_start, entry.new_hash); + fs.write_lines(file_path, lines, VCPKG_LINE_INFO); + + print2(Color::success, + "## The wrong sha in ", + stackTraceEntry.path, + ":", + stackTraceEntry.line, + " was successfully updated.\n"); + goto end_loop; + } + else + { + break; + } + } + } + print2(Color::error, + "Vcpkg was not able to automatically replace the hash ", + entry.old_hash, + " by ", + entry.new_hash, + " for the stacktrace \n", + entry.complete_stack_trace); + end_loop:; + } + } + }; + static ExtendedBuildResult do_build_package(const VcpkgCmdArguments& args, const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) @@ -895,17 +1080,22 @@ namespace vcpkg::Build auto stdoutlog = buildpath / ("stdout-" + action.spec.triplet().canonical_name() + ".log"); int return_code; { + Sha512Scanner scanner; auto out_file = fs.open_for_write(stdoutlog, VCPKG_LINE_INFO); return_code = cmd_execute_and_stream_data( command, [&](StringView sv) { print2(sv); + if (action.build_options.auto_update_mismatched_sha512 == Build::AutoUpdateMismatchedSHA512::YES) + scanner.parse_input(sv); Checks::check_exit(VCPKG_LINE_INFO, out_file.write(sv.data(), 1, sv.size()) == sv.size(), "Error occurred while writing '%s'", stdoutlog); }, env); + if (action.build_options.auto_update_mismatched_sha512 == Build::AutoUpdateMismatchedSHA512::YES) + scanner.replace_in_files(paths); } // close out_file // With the exception of empty packages, builds in "Download Mode" always result in failure. diff --git a/src/vcpkg/install.cpp b/src/vcpkg/install.cpp index f5a03a357f..7af15e45e2 100644 --- a/src/vcpkg/install.cpp +++ b/src/vcpkg/install.cpp @@ -525,6 +525,7 @@ namespace vcpkg::Install static constexpr StringLiteral OPTION_MANIFEST_FEATURE = "x-feature"; static constexpr StringLiteral OPTION_PROHIBIT_BACKCOMPAT_FEATURES = "x-prohibit-backcompat-features"; static constexpr StringLiteral OPTION_ALLOW_UNSUPPORTED_PORT = "allow-unsupported"; + static constexpr StringLiteral OPTION_AUTO_UPDATE_SHA512_HASHES = "x-auto-update-sha512-hashes"; static constexpr std::array INSTALL_SWITCHES = {{ {OPTION_DRY_RUN, "Do not actually build or install"}, @@ -544,6 +545,7 @@ namespace vcpkg::Install {OPTION_PROHIBIT_BACKCOMPAT_FEATURES, "(experimental) Fail install if a package attempts to use a deprecated feature"}, {OPTION_ALLOW_UNSUPPORTED_PORT, "Instead of erroring on an unsupported port, continue with a warning."}, + {OPTION_AUTO_UPDATE_SHA512_HASHES, "(experimental) Auto updates wrong sha512 file hashes in the portfiles"}, }}; static constexpr std::array MANIFEST_INSTALL_SWITCHES = {{ {OPTION_DRY_RUN, "Do not actually build or install"}, @@ -561,6 +563,7 @@ namespace vcpkg::Install {OPTION_CLEAN_DOWNLOADS_AFTER_BUILD, "Clean downloads after building each package"}, {OPTION_MANIFEST_NO_DEFAULT_FEATURES, "Don't install the default features from the manifest."}, {OPTION_ALLOW_UNSUPPORTED_PORT, "Instead of erroring on an unsupported port, continue with a warning."}, + {OPTION_AUTO_UPDATE_SHA512_HASHES, "(experimental) Auto updates wrong sha512 file hashes in the portfiles"}, }}; static constexpr std::array INSTALL_SETTINGS = {{ @@ -802,6 +805,7 @@ namespace vcpkg::Install const auto unsupported_port_action = Util::Sets::contains(options.switches, OPTION_ALLOW_UNSUPPORTED_PORT) ? Dependencies::UnsupportedPortAction::Warn : Dependencies::UnsupportedPortAction::Error; + const bool auto_update_512_hashes = Util::Sets::contains(options.switches, (OPTION_AUTO_UPDATE_SHA512_HASHES)); BinaryCache binary_cache; if (!only_downloads) @@ -826,6 +830,7 @@ namespace vcpkg::Install Build::PurgeDecompressFailure::NO, Util::Enum::to_enum(is_editable), prohibit_backcompat_features ? Build::BackcompatFeatures::PROHIBIT : Build::BackcompatFeatures::ALLOW, + Util::Enum::to_enum(auto_update_512_hashes), }; auto var_provider_storage = CMakeVars::make_triplet_cmake_var_provider(paths); From 8dd9b023135fca06b62a8892fed75f7529af74d7 Mon Sep 17 00:00:00 2001 From: Leander Schulten Date: Thu, 9 Sep 2021 04:46:01 +0200 Subject: [PATCH 3/4] install: Use new guid based approach --- include/vcpkg/base/downloads.h | 27 ++++- include/vcpkg/build.h | 4 +- src/vcpkg/base/downloads.cpp | 75 ++++++++---- src/vcpkg/build.cpp | 191 ++++++++++++++++++------------- src/vcpkg/commands.xdownload.cpp | 9 +- 5 files changed, 195 insertions(+), 111 deletions(-) diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h index 84753400ef..adc81ff245 100644 --- a/include/vcpkg/base/downloads.h +++ b/include/vcpkg/base/downloads.h @@ -23,10 +23,17 @@ namespace vcpkg::Downloads ExpectedS split_uri_view(StringView uri); } + enum class Sha512MismatchFormat + { + UserFriendly, + GuidWrapped, + }; + void verify_downloaded_file_hash(const Filesystem& fs, const std::string& sanitized_url, const Path& downloaded_path, - const std::string& sha512); + const std::string& sha512, + Sha512MismatchFormat mismatch_format = Sha512MismatchFormat::UserFriendly); View azure_blob_headers(); @@ -50,6 +57,13 @@ namespace vcpkg::Downloads Warn, Error, }; + + static constexpr StringLiteral guid_marker_hash_mismatch_start = "7279eda6-681f-46e0-aa5d-679ec14a2fb9"; + static constexpr StringLiteral guid_marker_hash_mismatch_end = "6982135f-5ad4-406f-86e3-f2e19c8966ef"; + // The following guids are used to detect the start and end of the output of the download command + static constexpr StringLiteral guid_marker_hash_mismatch_general_start = "b360a6a9-fb74-41de-a4c5-a7faf126d565"; + static constexpr StringLiteral guid_marker_hash_mismatch_general_end = "9d36a06a-0efa-470a-9a1e-63a26be67a84"; + // Handles downloading and uploading to a content addressable mirror struct DownloadManager { @@ -61,9 +75,10 @@ namespace vcpkg::Downloads const std::string& url, const Path& download_path, const Optional& sha512, - Sha512MismatchAction mismatch_action = Sha512MismatchAction::Error) const + Sha512MismatchAction mismatch_action = Sha512MismatchAction::Error, + Sha512MismatchFormat mismatch_format = Sha512MismatchFormat::UserFriendly) const { - this->download_file(fs, url, {}, download_path, sha512, mismatch_action); + this->download_file(fs, url, {}, download_path, sha512, mismatch_action, mismatch_format); } void download_file(Filesystem& fs, @@ -71,7 +86,8 @@ namespace vcpkg::Downloads View headers, const Path& download_path, const Optional& sha512, - Sha512MismatchAction mismatch_action = Sha512MismatchAction::Error) const; + Sha512MismatchAction mismatch_action = Sha512MismatchAction::Error, + Sha512MismatchFormat mismatch_format = Sha512MismatchFormat::UserFriendly) const; // Returns url that was successfully downloaded from std::string download_file(Filesystem& fs, @@ -79,7 +95,8 @@ namespace vcpkg::Downloads View headers, const Path& download_path, const Optional& sha512, - Sha512MismatchAction mismatch_action = Sha512MismatchAction::Error) const; + Sha512MismatchAction mismatch_action = Sha512MismatchAction::Error, + Sha512MismatchFormat mismatch_format = Sha512MismatchFormat::UserFriendly) const; ExpectedS put_file_to_mirror(const Filesystem& fs, const Path& file_to_put, StringView sha512) const; diff --git a/include/vcpkg/build.h b/include/vcpkg/build.h index ef48a036ec..011b8bb5c6 100644 --- a/include/vcpkg/build.h +++ b/include/vcpkg/build.h @@ -152,9 +152,9 @@ namespace vcpkg::Build YES }; - enum class AutoUpdateMismatchedSHA512 + enum class AutoUpdateMismatchedSHA512 : bool { - NO = 0, + NO, YES }; diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 17fe7156c8..6b05871371 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -186,23 +186,36 @@ namespace vcpkg::Downloads static std::string format_hash_mismatch(StringView url, const Path& downloaded_path, StringView expected, - StringView actual) + StringView actual, + Sha512MismatchFormat mismatch_format) { - return Strings::format("File does not have the expected hash:\n" - " url : [ %s ]\n" - " File path : [ %s ]\n" - " Expected hash : [ %s ]\n" - " Actual hash : [ %s ]\n", - url, - downloaded_path, + if (mismatch_format == Sha512MismatchFormat::UserFriendly) + { + return Strings::format("File does not have the expected hash:\n" + " url : [ %s ]\n" + " File path : [ %s ]\n" + " Expected hash : [ %s ]\n" + " Actual hash : [ %s ]\n", + url, + downloaded_path, + expected, + actual); + } + return Strings::format("%s\n" + "expected=%s\n" + "actual=%s\n" + "%s\n", + guid_marker_hash_mismatch_start, expected, - actual); + actual, + guid_marker_hash_mismatch_end); } static Optional try_verify_downloaded_file_hash(const Filesystem& fs, StringView sanitized_url, const Path& downloaded_path, - StringView sha512) + StringView sha512, + Sha512MismatchFormat mismatch_format) { std::string actual_hash = vcpkg::Hash::get_file_hash(VCPKG_LINE_INFO, fs, downloaded_path, Hash::Algorithm::Sha512); @@ -220,7 +233,7 @@ namespace vcpkg::Downloads if (sha512 != actual_hash) { - return format_hash_mismatch(sanitized_url, downloaded_path, sha512, actual_hash); + return format_hash_mismatch(sanitized_url, downloaded_path, sha512, actual_hash, mismatch_format); } return nullopt; } @@ -228,9 +241,10 @@ namespace vcpkg::Downloads void verify_downloaded_file_hash(const Filesystem& fs, const std::string& url, const Path& downloaded_path, - const std::string& sha512) + const std::string& sha512, + Sha512MismatchFormat mismatch_format) { - auto maybe_error = try_verify_downloaded_file_hash(fs, url, downloaded_path, sha512); + auto maybe_error = try_verify_downloaded_file_hash(fs, url, downloaded_path, sha512, mismatch_format); if (auto err = maybe_error.get()) { Checks::exit_with_message(VCPKG_LINE_INFO, *err); @@ -240,13 +254,15 @@ namespace vcpkg::Downloads static bool check_downloaded_file_hash(Filesystem& fs, const Optional& hash, Sha512MismatchAction mismatch_action, + Sha512MismatchFormat mismatch_format, StringView sanitized_url, const Path& download_part_path, std::string& errors) { if (auto p = hash.get()) { - auto maybe_error = try_verify_downloaded_file_hash(fs, sanitized_url, download_part_path, *p); + auto maybe_error = + try_verify_downloaded_file_hash(fs, sanitized_url, download_part_path, *p, mismatch_format); if (auto err = maybe_error.get()) { if (mismatch_action == Sha512MismatchAction::Warn) @@ -488,6 +504,7 @@ namespace vcpkg::Downloads const Path& download_path, const Optional& sha512, Sha512MismatchAction mismatch_action, + Sha512MismatchFormat mismatch_format, const std::vector& secrets, std::string& errors) { @@ -513,7 +530,7 @@ namespace vcpkg::Downloads if (download_winhttp(fs, download_path_part_path, split_uri, url, secrets, errors)) { if (check_downloaded_file_hash( - fs, sha512, only_warn_sha512_mismatch, url, download_path_part_path, errors)) + fs, sha512, mismatch_action, mismatch_format, url, download_path_part_path, errors)) { fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); return true; @@ -544,7 +561,8 @@ namespace vcpkg::Downloads return false; } - if (check_downloaded_file_hash(fs, sha512, mismatch_action, sanitized_url, download_path_part_path, errors)) + if (check_downloaded_file_hash( + fs, sha512, mismatch_action, mismatch_format, sanitized_url, download_path_part_path, errors)) { fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); return true; @@ -558,12 +576,14 @@ namespace vcpkg::Downloads const Path& download_path, const Optional& sha512, Sha512MismatchAction mismatch_action, + Sha512MismatchFormat mismatch_format, const std::vector& secrets, std::string& errors) { for (auto&& url : urls) { - if (try_download_file(fs, url, headers, download_path, sha512, mismatch_action, secrets, errors)) + if (try_download_file( + fs, url, headers, download_path, sha512, mismatch_action, mismatch_format, secrets, errors)) return url; } return nullopt; @@ -580,9 +600,11 @@ namespace vcpkg::Downloads View headers, const Path& download_path, const Optional& sha512, - Sha512MismatchAction mismatch_action) const + Sha512MismatchAction mismatch_action, + Sha512MismatchFormat mismatch_format) const { - this->download_file(fs, View(&url, 1), headers, download_path, sha512, mismatch_action); + this->download_file( + fs, View(&url, 1), headers, download_path, sha512, mismatch_action, mismatch_format); } std::string DownloadManager::download_file(Filesystem& fs, @@ -590,7 +612,8 @@ namespace vcpkg::Downloads View headers, const Path& download_path, const Optional& sha512, - Sha512MismatchAction mismatch_action) const + Sha512MismatchAction mismatch_action, + Sha512MismatchFormat mismatch_format) const { std::string errors; if (auto hash = sha512.get()) @@ -604,6 +627,7 @@ namespace vcpkg::Downloads download_path, sha512, mismatch_action, + mismatch_format, m_config.m_secrets, errors)) return read_url; @@ -625,8 +649,15 @@ namespace vcpkg::Downloads } else { - auto maybe_url = try_download_files( - fs, urls, headers, download_path, sha512, mismatch_action, m_config.m_secrets, errors); + auto maybe_url = try_download_files(fs, + urls, + headers, + download_path, + sha512, + mismatch_action, + mismatch_format, + m_config.m_secrets, + errors); if (auto url = maybe_url.get()) { if (auto hash = sha512.get()) diff --git a/src/vcpkg/build.cpp b/src/vcpkg/build.cpp index 1e72c81e4e..f87115292e 100644 --- a/src/vcpkg/build.cpp +++ b/src/vcpkg/build.cpp @@ -866,82 +866,96 @@ namespace vcpkg::Build std::string complete_stack_trace; std::vector stack_trace; }; - std::vector entries; Entry current_entry; + std::string captured_output; enum class State { None, Start, + Hashes, + HaveHashes, CallStack } state = State::None; + const VcpkgPaths& paths; public: - void parse_input(StringView input) + Sha512Scanner(const VcpkgPaths& paths) : paths(paths) { } + void process_input(StringView input) { if (input == "\n") { - parse_line(""); + process_line(""); } else { - for (const auto line : Strings::split(input, '\n')) + for (const auto& line : Strings::split(input, '\n')) { - parse_line(line); + process_line(line); } } } private: - void parse_line(const std::string& line) + void process_line(const std::string& line) { - auto index = std::string::npos; - if (Strings::starts_with(line, "File does not have the expected hash")) + if (Strings::equals(line, Downloads::guid_marker_hash_mismatch_general_start)) { state = State::Start; + return; // don't capture output + } + if (Strings::equals(line, Downloads::guid_marker_hash_mismatch_general_end)) + { + if (state != State::CallStack) + { + print2(Color::error, + "Vcpkg was not able to automatically replace the SHA512 hash. Raw output:", + captured_output); + } + else + { + replace_in_files(); + } + current_entry = Entry(); + captured_output.clear(); + state = State::None; + return; // don't capture output } else if (state == State::None) { - return; // don't check every line for performance reasons + print2(line, '\n'); + return; // don't capture output } - else if ((index = line.find("Expected hash : [ ")) != std::string::npos) + else if (state == State::Start && Strings::equals(line, Downloads::guid_marker_hash_mismatch_start)) { - index += 18; // length of "Expected hash : [ " - if (line.length() < index + 128) - { - state = State::None; - return; - } - current_entry.old_hash = line.substr(index, 128); + state = State::Hashes; } - else if ((index = line.find("Actual hash : [ ")) != std::string::npos) + else if (state == State::Hashes) { - index += 16; // length of "Actual hash : [ " - if (line.length() < index + 128) - { - state = State::None; - return; - } - current_entry.new_hash = line.substr(index, 128); + if (Strings::equals(line, Downloads::guid_marker_hash_mismatch_end)) + state = State::HaveHashes; + else if (Strings::starts_with(line, "expected=")) + current_entry.old_hash = Strings::trim(line.substr(9 /* length of 'expected=' */)); + else if (Strings::starts_with(line, "actual=")) + current_entry.new_hash = Strings::trim(line.substr(7 /* length of 'actual=' */)); } - else if (line.find("Call Stack (most recent call first):") != std::string::npos) + else if (state == State::HaveHashes && + line.find("Call Stack (most recent call first):") != std::string::npos) { state = State::CallStack; } else if (state == State::CallStack) { - if (line.empty()) + if (!line.empty()) { - state = State::None; - entries.push_back(current_entry); - current_entry = Entry(); - return; + current_entry.complete_stack_trace += line; + current_entry.complete_stack_trace += '\n'; + if (line.length() < 5) // a real entry can not only constis out of 5 chars + return; + parse_stack_trace_entry(line.substr(2)); } - current_entry.complete_stack_trace += line; - current_entry.complete_stack_trace += '\n'; - if (line.length() < 5) // a real entry can not only constis out of 5 chars - return; - parse_stack_trace_entry(line.substr(2)); } + captured_output += line; + captured_output += '\n'; } void parse_stack_trace_entry(std::string line) @@ -982,55 +996,71 @@ namespace vcpkg::Build } public: - void replace_in_files(const VcpkgPaths& paths) + void replace_in_files() { auto& fs = paths.get_filesystem(); - for (const auto& entry : entries) + + for (const auto& stackTraceEntry : current_entry.stack_trace) { - for (const auto& stackTraceEntry : entry.stack_trace) + const auto file_path = paths.root / stackTraceEntry.path; + auto lines = fs.read_lines(file_path, VCPKG_LINE_INFO); + for (int i = stackTraceEntry.line - 1; + i < std::min(stackTraceEntry.line + 10, static_cast(lines.size())); + ++i) { - const auto file_path = paths.root / stackTraceEntry.path; - auto lines = fs.read_lines(file_path, VCPKG_LINE_INFO); - for (int i = stackTraceEntry.line - 1; - i < std::min(stackTraceEntry.line + 10, static_cast(lines.size())); - ++i) - { - auto& line = lines[i]; - const auto sha512_index = line.find("SHA512"); - if (sha512_index == std::string::npos) continue; - const auto sha_start = line.find_first_not_of(" ", sha512_index + 6); - if (sha_start == std::string::npos) continue; - auto sha_end = line.find_first_not_of("0123456789abcdef", sha_start); - if (sha_end == std::string::npos) sha_end = line.size(); - const auto sha = line.substr(sha_start, sha_end); - if (sha == "0" || sha == entry.old_hash) - { - line.replace(sha_start, sha_end - sha_start, entry.new_hash); - fs.write_lines(file_path, lines, VCPKG_LINE_INFO); - - print2(Color::success, - "## The wrong sha in ", - stackTraceEntry.path, - ":", - stackTraceEntry.line, - " was successfully updated.\n"); - goto end_loop; - } - else + auto& currentLine = lines[i]; + const auto sha512_index = currentLine.find("SHA512"); + if (sha512_index == std::string::npos) continue; + const auto sha_start = currentLine.find_first_not_of(' ', sha512_index + 6); + if (sha_start == std::string::npos) continue; + if (currentLine[sha_start] == '"') // found something like `set(HASH "abc123")` `SHA512 ${HASH}` + { // search for the old hash in the whole file and replace it + if (current_entry.old_hash == "0") break; // don't replace any 0 + for (auto& line : lines) { - break; + const auto start = line.find(current_entry.old_hash); + if (start != std::string::npos) + { + line.replace(start, current_entry.old_hash.size(), current_entry.new_hash); + fs.write_lines(file_path, lines, VCPKG_LINE_INFO); + + print2(Color::success, + "## The wrong SHA512 in ", + stackTraceEntry.path, + " was successfully updated.\n"); + return; + } } } + auto sha_end = currentLine.find_first_not_of("0123456789abcdef", sha_start); + if (sha_end == std::string::npos) sha_end = currentLine.size(); + const auto sha = currentLine.substr(sha_start, sha_end); + if (sha == "0" || sha == current_entry.old_hash) + { + currentLine.replace(sha_start, sha_end - sha_start, current_entry.new_hash); + fs.write_lines(file_path, lines, VCPKG_LINE_INFO); + + print2(Color::success, + "## The wrong SHA512 in ", + stackTraceEntry.path, + ":", + stackTraceEntry.line, + " was successfully updated.\n"); + return; + } + else + { + break; + } } - print2(Color::error, - "Vcpkg was not able to automatically replace the hash ", - entry.old_hash, - " by ", - entry.new_hash, - " for the stacktrace \n", - entry.complete_stack_trace); - end_loop:; } + print2(Color::error, + "vcpkg was not able to automatically replace the hash ", + current_entry.old_hash, + " by ", + current_entry.new_hash, + " for the stacktrace \n", + current_entry.complete_stack_trace); } }; @@ -1080,22 +1110,21 @@ namespace vcpkg::Build auto stdoutlog = buildpath / ("stdout-" + action.spec.triplet().canonical_name() + ".log"); int return_code; { - Sha512Scanner scanner; + Sha512Scanner scanner(paths); auto out_file = fs.open_for_write(stdoutlog, VCPKG_LINE_INFO); return_code = cmd_execute_and_stream_data( command, [&](StringView sv) { - print2(sv); if (action.build_options.auto_update_mismatched_sha512 == Build::AutoUpdateMismatchedSHA512::YES) - scanner.parse_input(sv); + scanner.process_input(sv); + else + print2(sv); Checks::check_exit(VCPKG_LINE_INFO, out_file.write(sv.data(), 1, sv.size()) == sv.size(), "Error occurred while writing '%s'", stdoutlog); }, env); - if (action.build_options.auto_update_mismatched_sha512 == Build::AutoUpdateMismatchedSHA512::YES) - scanner.replace_in_files(paths); } // close out_file // With the exception of empty packages, builds in "Download Mode" always result in failure. diff --git a/src/vcpkg/commands.xdownload.cpp b/src/vcpkg/commands.xdownload.cpp index dd733270e9..e261cc65f6 100644 --- a/src/vcpkg/commands.xdownload.cpp +++ b/src/vcpkg/commands.xdownload.cpp @@ -15,6 +15,7 @@ namespace vcpkg::Commands::X_Download static constexpr StringLiteral OPTION_SKIP_SHA512 = "skip-sha512"; static constexpr StringLiteral OPTION_SHA512 = "sha512"; static constexpr StringLiteral OPTION_ONLY_WARN_FOR_SHA512_MISMATCH = "only-warn-sha512-mismatch"; + static constexpr StringLiteral OPTION_SHA512_MISMATCH_GUID_WRAPPED = "sha512-mismatch-guid-wrapped"; static constexpr StringLiteral OPTION_URL = "url"; static constexpr StringLiteral OPTION_HEADER = "header"; @@ -23,6 +24,8 @@ namespace vcpkg::Commands::X_Download {OPTION_SKIP_SHA512, "Do not check the SHA512 of the downloaded file"}, {OPTION_ONLY_WARN_FOR_SHA512_MISMATCH, "Only warn if the SHA512 of the downloaded file mismatched instead of fail"}, + {OPTION_SHA512_MISMATCH_GUID_WRAPPED, + "Instead of displaying a user friendly message for a hash mismatch print a mashine readable one"}, }; static constexpr CommandSetting FETCH_SETTINGS[] = { {OPTION_SHA512, "The hash of the file to be downloaded"}, @@ -105,6 +108,10 @@ namespace vcpkg::Commands::X_Download using Action = Downloads::Sha512MismatchAction; auto mismatch_action = Util::Sets::contains(parsed.switches, OPTION_ONLY_WARN_FOR_SHA512_MISMATCH) ? Action::Warn : Action::Error; + using Format = Downloads::Sha512MismatchFormat; + auto mismatch_format = Util::Sets::contains(parsed.switches, OPTION_SHA512_MISMATCH_GUID_WRAPPED) + ? Format::GuidWrapped + : Format::UserFriendly; // Is this a store command? if (Util::Sets::contains(parsed.switches, OPTION_STORE)) @@ -145,7 +152,7 @@ namespace vcpkg::Commands::X_Download urls = it_urls->second; } - download_manager.download_file(fs, urls, headers, file, sha, mismatch_action); + download_manager.download_file(fs, urls, headers, file, sha, mismatch_action, mismatch_format); Checks::exit_success(VCPKG_LINE_INFO); } } From 9282cfe8fbd8bfb4ff100a5cd7164f17424bbb64 Mon Sep 17 00:00:00 2001 From: autoantwort <41973254+autoantwort@users.noreply.github.com> Date: Tue, 28 Sep 2021 23:56:02 +0200 Subject: [PATCH 4/4] Apply suggestions from code review Co-authored-by: nicole mazzuca <83086508+strega-nil-ms@users.noreply.github.com> --- src/vcpkg/build.cpp | 48 ++++++++++++++++---------------- src/vcpkg/commands.xdownload.cpp | 4 +-- src/vcpkg/install.cpp | 4 +-- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/vcpkg/build.cpp b/src/vcpkg/build.cpp index f87115292e..4979aff299 100644 --- a/src/vcpkg/build.cpp +++ b/src/vcpkg/build.cpp @@ -949,7 +949,7 @@ namespace vcpkg::Build { current_entry.complete_stack_trace += line; current_entry.complete_stack_trace += '\n'; - if (line.length() < 5) // a real entry can not only constis out of 5 chars + if (line.length() < 5) // a real entry can not only consist of 5 chars return; parse_stack_trace_entry(line.substr(2)); } @@ -961,21 +961,21 @@ namespace vcpkg::Build void parse_stack_trace_entry(std::string line) { auto colon = line.find(":"); - const auto printError = [&] { + const auto print_error = [&] { print2(Color::error, - "The Strack trace line should have the format `path/fileName.cmake: " - "(functionName)` but is ", + "The stacktrace line should have the format 'path/file_name.cmake: " + "(function_name)' but is ", line); }; if (colon == std::string::npos) { - printError(); + print_error(); return; } auto space = line.find(" ", colon); if (space == std::string::npos) { - printError(); + print_error(); return; } const auto path = line.substr(0, colon); @@ -990,7 +990,7 @@ namespace vcpkg::Build } catch (std::exception&) { - printError(); + print_error(); return; } } @@ -1000,21 +1000,21 @@ namespace vcpkg::Build { auto& fs = paths.get_filesystem(); - for (const auto& stackTraceEntry : current_entry.stack_trace) + for (const auto& stacktrace_entry : current_entry.stack_trace) { - const auto file_path = paths.root / stackTraceEntry.path; + const auto file_path = paths.root / stacktrace_entry.path; auto lines = fs.read_lines(file_path, VCPKG_LINE_INFO); - for (int i = stackTraceEntry.line - 1; - i < std::min(stackTraceEntry.line + 10, static_cast(lines.size())); + for (int i = stacktrace_entry.line - 1; + i < std::min(stacktrace_entry.line + 10, static_cast(lines.size())); ++i) { - auto& currentLine = lines[i]; - const auto sha512_index = currentLine.find("SHA512"); + auto& current_line = lines[i]; + const auto sha512_index = current_line.find("SHA512"); if (sha512_index == std::string::npos) continue; - const auto sha_start = currentLine.find_first_not_of(' ', sha512_index + 6); + const auto sha_start = current_line.find_first_not_of(' ', sha512_index + 6); if (sha_start == std::string::npos) continue; - if (currentLine[sha_start] == '"') // found something like `set(HASH "abc123")` `SHA512 ${HASH}` - { // search for the old hash in the whole file and replace it + if (current_line[sha_start] == '"') // found something like `set(HASH "abc123")` `SHA512 ${HASH}` + { // search for the old hash in the whole file and replace it if (current_entry.old_hash == "0") break; // don't replace any 0 for (auto& line : lines) { @@ -1026,25 +1026,25 @@ namespace vcpkg::Build print2(Color::success, "## The wrong SHA512 in ", - stackTraceEntry.path, + stacktrace_entry.path, " was successfully updated.\n"); return; } } } - auto sha_end = currentLine.find_first_not_of("0123456789abcdef", sha_start); - if (sha_end == std::string::npos) sha_end = currentLine.size(); - const auto sha = currentLine.substr(sha_start, sha_end); + auto sha_end = current_line.find_first_not_of("0123456789abcdef", sha_start); + if (sha_end == std::string::npos) sha_end = current_line.size(); + const auto sha = current_line.substr(sha_start, sha_end); if (sha == "0" || sha == current_entry.old_hash) { - currentLine.replace(sha_start, sha_end - sha_start, current_entry.new_hash); + current_line.replace(sha_start, sha_end - sha_start, current_entry.new_hash); fs.write_lines(file_path, lines, VCPKG_LINE_INFO); print2(Color::success, "## The wrong SHA512 in ", - stackTraceEntry.path, + stacktrace_entry.path, ":", - stackTraceEntry.line, + stacktrace_entry.line, " was successfully updated.\n"); return; } @@ -1110,7 +1110,7 @@ namespace vcpkg::Build auto stdoutlog = buildpath / ("stdout-" + action.spec.triplet().canonical_name() + ".log"); int return_code; { - Sha512Scanner scanner(paths); + Sha512Scanner scanner{paths}; auto out_file = fs.open_for_write(stdoutlog, VCPKG_LINE_INFO); return_code = cmd_execute_and_stream_data( command, diff --git a/src/vcpkg/commands.xdownload.cpp b/src/vcpkg/commands.xdownload.cpp index e261cc65f6..302a5d8be0 100644 --- a/src/vcpkg/commands.xdownload.cpp +++ b/src/vcpkg/commands.xdownload.cpp @@ -23,9 +23,9 @@ namespace vcpkg::Commands::X_Download {OPTION_STORE, "Indicates the file should be stored instead of fetched"}, {OPTION_SKIP_SHA512, "Do not check the SHA512 of the downloaded file"}, {OPTION_ONLY_WARN_FOR_SHA512_MISMATCH, - "Only warn if the SHA512 of the downloaded file mismatched instead of fail"}, + "Only warn if the SHA512 of the downloaded file was incorrect, instead of failing"}, {OPTION_SHA512_MISMATCH_GUID_WRAPPED, - "Instead of displaying a user friendly message for a hash mismatch print a mashine readable one"}, + "Instead of displaying a user friendly message for a hash mismatch, print a machine readable one"}, }; static constexpr CommandSetting FETCH_SETTINGS[] = { {OPTION_SHA512, "The hash of the file to be downloaded"}, diff --git a/src/vcpkg/install.cpp b/src/vcpkg/install.cpp index 7af15e45e2..61e5b8131b 100644 --- a/src/vcpkg/install.cpp +++ b/src/vcpkg/install.cpp @@ -527,7 +527,7 @@ namespace vcpkg::Install static constexpr StringLiteral OPTION_ALLOW_UNSUPPORTED_PORT = "allow-unsupported"; static constexpr StringLiteral OPTION_AUTO_UPDATE_SHA512_HASHES = "x-auto-update-sha512-hashes"; - static constexpr std::array INSTALL_SWITCHES = {{ + static constexpr std::array INSTALL_SWITCHES = {{ {OPTION_DRY_RUN, "Do not actually build or install"}, {OPTION_USE_HEAD_VERSION, "Install the libraries on the command line using the latest upstream sources"}, {OPTION_NO_DOWNLOADS, "Do not download new sources"}, @@ -547,7 +547,7 @@ namespace vcpkg::Install {OPTION_ALLOW_UNSUPPORTED_PORT, "Instead of erroring on an unsupported port, continue with a warning."}, {OPTION_AUTO_UPDATE_SHA512_HASHES, "(experimental) Auto updates wrong sha512 file hashes in the portfiles"}, }}; - static constexpr std::array MANIFEST_INSTALL_SWITCHES = {{ + static constexpr std::array MANIFEST_INSTALL_SWITCHES = {{ {OPTION_DRY_RUN, "Do not actually build or install"}, {OPTION_USE_HEAD_VERSION, "Install the libraries on the command line using the latest upstream sources"}, {OPTION_NO_DOWNLOADS, "Do not download new sources"},