diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h index c89d5370bd..adc81ff245 100644 --- a/include/vcpkg/base/downloads.h +++ b/include/vcpkg/base/downloads.h @@ -23,10 +23,17 @@ namespace vcpkg::Downloads ExpectedS split_uri_view(StringView uri); } + enum class Sha512MismatchFormat + { + UserFriendly, + GuidWrapped, + }; + void verify_downloaded_file_hash(const Filesystem& fs, const std::string& sanitized_url, const Path& downloaded_path, - const std::string& sha512); + const std::string& sha512, + Sha512MismatchFormat mismatch_format = Sha512MismatchFormat::UserFriendly); View azure_blob_headers(); @@ -45,6 +52,18 @@ namespace vcpkg::Downloads bool m_block_origin = false; }; + enum class Sha512MismatchAction : bool + { + Warn, + Error, + }; + + static constexpr StringLiteral guid_marker_hash_mismatch_start = "7279eda6-681f-46e0-aa5d-679ec14a2fb9"; + static constexpr StringLiteral guid_marker_hash_mismatch_end = "6982135f-5ad4-406f-86e3-f2e19c8966ef"; + // The following guids are used to detect the start and end of the output of the download command + static constexpr StringLiteral guid_marker_hash_mismatch_general_start = "b360a6a9-fb74-41de-a4c5-a7faf126d565"; + static constexpr StringLiteral guid_marker_hash_mismatch_general_end = "9d36a06a-0efa-470a-9a1e-63a26be67a84"; + // Handles downloading and uploading to a content addressable mirror struct DownloadManager { @@ -55,23 +74,29 @@ namespace vcpkg::Downloads void download_file(Filesystem& fs, const std::string& url, const Path& download_path, - const Optional& sha512) const + const Optional& sha512, + Sha512MismatchAction mismatch_action = Sha512MismatchAction::Error, + Sha512MismatchFormat mismatch_format = Sha512MismatchFormat::UserFriendly) const { - this->download_file(fs, url, {}, download_path, sha512); + this->download_file(fs, url, {}, download_path, sha512, mismatch_action, mismatch_format); } void download_file(Filesystem& fs, const std::string& url, View headers, const Path& download_path, - const Optional& sha512) const; + const Optional& sha512, + Sha512MismatchAction mismatch_action = Sha512MismatchAction::Error, + Sha512MismatchFormat mismatch_format = Sha512MismatchFormat::UserFriendly) const; // Returns url that was successfully downloaded from std::string download_file(Filesystem& fs, View urls, View headers, const Path& download_path, - const Optional& sha512) const; + const Optional& sha512, + Sha512MismatchAction mismatch_action = Sha512MismatchAction::Error, + Sha512MismatchFormat mismatch_format = Sha512MismatchFormat::UserFriendly) const; ExpectedS put_file_to_mirror(const Filesystem& fs, const Path& file_to_put, StringView sha512) const; diff --git a/include/vcpkg/build.h b/include/vcpkg/build.h index 539cafa986..581c35bd4e 100644 --- a/include/vcpkg/build.h +++ b/include/vcpkg/build.h @@ -152,6 +152,12 @@ namespace vcpkg::Build YES }; + enum class AutoUpdateMismatchedSHA512 : bool + { + NO, + YES + }; + struct BuildPackageOptions { BuildMissing build_missing; @@ -165,6 +171,7 @@ namespace vcpkg::Build PurgeDecompressFailure purge_decompress_failure; Editable editable; BackcompatFeatures backcompat_features; + AutoUpdateMismatchedSHA512 auto_update_mismatched_sha512; }; static constexpr BuildPackageOptions default_build_package_options{ diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index bf29d2a466..3979af28c9 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -189,23 +189,36 @@ namespace vcpkg::Downloads static std::string format_hash_mismatch(StringView url, const Path& downloaded_path, StringView expected, - StringView actual) + StringView actual, + Sha512MismatchFormat mismatch_format) { - return Strings::format("File does not have the expected hash:\n" - " url : [ %s ]\n" - " File path : [ %s ]\n" - " Expected hash : [ %s ]\n" - " Actual hash : [ %s ]\n", - url, - downloaded_path, + if (mismatch_format == Sha512MismatchFormat::UserFriendly) + { + return Strings::format("File does not have the expected hash:\n" + " url : [ %s ]\n" + " File path : [ %s ]\n" + " Expected hash : [ %s ]\n" + " Actual hash : [ %s ]\n", + url, + downloaded_path, + expected, + actual); + } + return Strings::format("%s\n" + "expected=%s\n" + "actual=%s\n" + "%s\n", + guid_marker_hash_mismatch_start, expected, - actual); + actual, + guid_marker_hash_mismatch_end); } static Optional try_verify_downloaded_file_hash(const Filesystem& fs, StringView sanitized_url, const Path& downloaded_path, - StringView sha512) + StringView sha512, + Sha512MismatchFormat mismatch_format) { std::string actual_hash = vcpkg::Hash::get_file_hash(VCPKG_LINE_INFO, fs, downloaded_path, Hash::Algorithm::Sha512); @@ -223,7 +236,7 @@ namespace vcpkg::Downloads if (sha512 != actual_hash) { - return format_hash_mismatch(sanitized_url, downloaded_path, sha512, actual_hash); + return format_hash_mismatch(sanitized_url, downloaded_path, sha512, actual_hash, mismatch_format); } return nullopt; } @@ -231,9 +244,10 @@ namespace vcpkg::Downloads void verify_downloaded_file_hash(const Filesystem& fs, const std::string& url, const Path& downloaded_path, - const std::string& sha512) + const std::string& sha512, + Sha512MismatchFormat mismatch_format) { - auto maybe_error = try_verify_downloaded_file_hash(fs, url, downloaded_path, sha512); + auto maybe_error = try_verify_downloaded_file_hash(fs, url, downloaded_path, sha512, mismatch_format); if (auto err = maybe_error.get()) { Checks::exit_with_message(VCPKG_LINE_INFO, *err); @@ -242,15 +256,23 @@ namespace vcpkg::Downloads static bool check_downloaded_file_hash(Filesystem& fs, const Optional& hash, + Sha512MismatchAction mismatch_action, + Sha512MismatchFormat mismatch_format, StringView sanitized_url, const Path& download_part_path, std::string& errors) { if (auto p = hash.get()) { - auto maybe_error = try_verify_downloaded_file_hash(fs, sanitized_url, download_part_path, *p); + auto maybe_error = + try_verify_downloaded_file_hash(fs, sanitized_url, download_part_path, *p, mismatch_format); if (auto err = maybe_error.get()) { + if (mismatch_action == Sha512MismatchAction::Warn) + { + print2(Color::warning, *err, '\n'); + return true; + } Strings::append(errors, *err, '\n'); return false; } @@ -511,6 +533,8 @@ namespace vcpkg::Downloads View headers, const Path& download_path, const Optional& sha512, + Sha512MismatchAction mismatch_action, + Sha512MismatchFormat mismatch_format, const std::vector& secrets, std::string& errors) { @@ -535,7 +559,8 @@ namespace vcpkg::Downloads { if (download_winhttp(fs, download_path_part_path, split_uri, url, secrets, errors)) { - if (check_downloaded_file_hash(fs, sha512, url, download_path_part_path, errors)) + if (check_downloaded_file_hash( + fs, sha512, mismatch_action, mismatch_format, url, download_path_part_path, errors)) { fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); return true; @@ -566,7 +591,8 @@ namespace vcpkg::Downloads return false; } - if (check_downloaded_file_hash(fs, sha512, sanitized_url, download_path_part_path, errors)) + if (check_downloaded_file_hash( + fs, sha512, mismatch_action, mismatch_format, sanitized_url, download_path_part_path, errors)) { fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); return true; @@ -579,12 +605,16 @@ namespace vcpkg::Downloads View headers, const Path& download_path, const Optional& sha512, + Sha512MismatchAction mismatch_action, + Sha512MismatchFormat mismatch_format, const std::vector& secrets, std::string& errors) { for (auto&& url : urls) { - if (try_download_file(fs, url, headers, download_path, sha512, secrets, errors)) return url; + if (try_download_file( + fs, url, headers, download_path, sha512, mismatch_action, mismatch_format, secrets, errors)) + return url; } return nullopt; } @@ -599,16 +629,21 @@ namespace vcpkg::Downloads const std::string& url, View headers, const Path& download_path, - const Optional& sha512) const + const Optional& sha512, + Sha512MismatchAction mismatch_action, + Sha512MismatchFormat mismatch_format) const { - this->download_file(fs, View(&url, 1), headers, download_path, sha512); + this->download_file( + fs, View(&url, 1), headers, download_path, sha512, mismatch_action, mismatch_format); } std::string DownloadManager::download_file(Filesystem& fs, View urls, View headers, const Path& download_path, - const Optional& sha512) const + const Optional& sha512, + Sha512MismatchAction mismatch_action, + Sha512MismatchFormat mismatch_format) const { std::string errors; if (auto hash = sha512.get()) @@ -616,8 +651,15 @@ namespace vcpkg::Downloads if (auto read_template = m_config.m_read_url_template.get()) { auto read_url = Strings::replace_all(*read_template, "", *hash); - if (Downloads::try_download_file( - fs, read_url, m_config.m_read_headers, download_path, sha512, m_config.m_secrets, errors)) + if (Downloads::try_download_file(fs, + read_url, + m_config.m_read_headers, + download_path, + sha512, + mismatch_action, + mismatch_format, + m_config.m_secrets, + errors)) return read_url; } } @@ -637,8 +679,15 @@ namespace vcpkg::Downloads } else { - auto maybe_url = - try_download_files(fs, urls, headers, download_path, sha512, m_config.m_secrets, errors); + auto maybe_url = try_download_files(fs, + urls, + headers, + download_path, + sha512, + mismatch_action, + mismatch_format, + m_config.m_secrets, + errors); if (auto url = maybe_url.get()) { if (auto hash = sha512.get()) diff --git a/src/vcpkg/build.cpp b/src/vcpkg/build.cpp index f822b4db16..6c4e8c4352 100644 --- a/src/vcpkg/build.cpp +++ b/src/vcpkg/build.cpp @@ -750,6 +750,8 @@ namespace vcpkg::Build {"_VCPKG_DOWNLOAD_TOOL", to_string(action.build_options.download_tool)}, {"_VCPKG_EDITABLE", Util::Enum::to_bool(action.build_options.editable) ? "1" : "0"}, {"_VCPKG_NO_DOWNLOADS", !Util::Enum::to_bool(action.build_options.allow_downloads) ? "1" : "0"}, + {"Z_VCPKG_SHA512_MISMATCH_NO_ERROR", + Util::Enum::to_bool(action.build_options.auto_update_mismatched_sha512) ? "1" : "0"}, }; if (action.build_options.download_tool == DownloadTool::ARIA2) @@ -854,6 +856,219 @@ namespace vcpkg::Build } } + struct Sha512Scanner + { + private: + struct Entry + { + std::string old_hash; + std::string new_hash; + struct StackTraceEntry + { + std::string path; + int line; + }; + std::string complete_stack_trace; + std::vector stack_trace; + }; + Entry current_entry; + std::string captured_output; + enum class State + { + None, + Start, + Hashes, + HaveHashes, + CallStack + } state = State::None; + const VcpkgPaths& paths; + + public: + Sha512Scanner(const VcpkgPaths& paths) : paths(paths) { } + void process_input(StringView input) + { + if (input == "\n") + { + process_line(""); + } + else + { + for (const auto& line : Strings::split(input, '\n')) + { + process_line(line); + } + } + } + + private: + void process_line(const std::string& line) + { + if (Strings::equals(line, Downloads::guid_marker_hash_mismatch_general_start)) + { + state = State::Start; + return; // don't capture output + } + if (Strings::equals(line, Downloads::guid_marker_hash_mismatch_general_end)) + { + if (state != State::CallStack) + { + print2(Color::error, + "Vcpkg was not able to automatically replace the SHA512 hash. Raw output:", + captured_output); + } + else + { + replace_in_files(); + } + current_entry = Entry(); + captured_output.clear(); + state = State::None; + return; // don't capture output + } + else if (state == State::None) + { + print2(line, '\n'); + return; // don't capture output + } + else if (state == State::Start && Strings::equals(line, Downloads::guid_marker_hash_mismatch_start)) + { + state = State::Hashes; + } + else if (state == State::Hashes) + { + if (Strings::equals(line, Downloads::guid_marker_hash_mismatch_end)) + state = State::HaveHashes; + else if (Strings::starts_with(line, "expected=")) + current_entry.old_hash = Strings::trim(line.substr(9 /* length of 'expected=' */)); + else if (Strings::starts_with(line, "actual=")) + current_entry.new_hash = Strings::trim(line.substr(7 /* length of 'actual=' */)); + } + else if (state == State::HaveHashes && + line.find("Call Stack (most recent call first):") != std::string::npos) + { + state = State::CallStack; + } + else if (state == State::CallStack) + { + if (!line.empty()) + { + current_entry.complete_stack_trace += line; + current_entry.complete_stack_trace += '\n'; + if (line.length() < 5) // a real entry can not only consist of 5 chars + return; + parse_stack_trace_entry(line.substr(2)); + } + } + captured_output += line; + captured_output += '\n'; + } + + void parse_stack_trace_entry(std::string line) + { + auto colon = line.find(":"); + const auto print_error = [&] { + print2(Color::error, + "The stacktrace line should have the format 'path/file_name.cmake: " + "(function_name)' but is ", + line); + }; + if (colon == std::string::npos) + { + print_error(); + return; + } + auto space = line.find(" ", colon); + if (space == std::string::npos) + { + print_error(); + return; + } + const auto path = line.substr(0, colon); + if (Strings::starts_with(path, "scripts/")) + { + return; + } + try + { + auto line_number = std::stoi(line.substr(colon + 1, space)); + current_entry.stack_trace.push_back(Entry::StackTraceEntry{path, line_number}); + } + catch (std::exception&) + { + print_error(); + return; + } + } + + public: + void replace_in_files() + { + auto& fs = paths.get_filesystem(); + + for (const auto& stacktrace_entry : current_entry.stack_trace) + { + const auto file_path = paths.root / stacktrace_entry.path; + auto lines = fs.read_lines(file_path, VCPKG_LINE_INFO); + for (int i = stacktrace_entry.line - 1; + i < std::min(stacktrace_entry.line + 10, static_cast(lines.size())); + ++i) + { + auto& current_line = lines[i]; + const auto sha512_index = current_line.find("SHA512"); + if (sha512_index == std::string::npos) continue; + const auto sha_start = current_line.find_first_not_of(' ', sha512_index + 6); + if (sha_start == std::string::npos) continue; + if (current_line[sha_start] == '"') // found something like `set(HASH "abc123")` `SHA512 ${HASH}` + { // search for the old hash in the whole file and replace it + if (current_entry.old_hash == "0") break; // don't replace any 0 + for (auto& line : lines) + { + const auto start = line.find(current_entry.old_hash); + if (start != std::string::npos) + { + line.replace(start, current_entry.old_hash.size(), current_entry.new_hash); + fs.write_lines(file_path, lines, VCPKG_LINE_INFO); + + print2(Color::success, + "## The wrong SHA512 in ", + stacktrace_entry.path, + " was successfully updated.\n"); + return; + } + } + } + auto sha_end = current_line.find_first_not_of("0123456789abcdef", sha_start); + if (sha_end == std::string::npos) sha_end = current_line.size(); + const auto sha = current_line.substr(sha_start, sha_end); + if (sha == "0" || sha == current_entry.old_hash) + { + current_line.replace(sha_start, sha_end - sha_start, current_entry.new_hash); + fs.write_lines(file_path, lines, VCPKG_LINE_INFO); + + print2(Color::success, + "## The wrong SHA512 in ", + stacktrace_entry.path, + ":", + stacktrace_entry.line, + " was successfully updated.\n"); + return; + } + else + { + break; + } + } + } + print2(Color::error, + "vcpkg was not able to automatically replace the hash ", + current_entry.old_hash, + " by ", + current_entry.new_hash, + " for the stacktrace \n", + current_entry.complete_stack_trace); + } + }; + static ExtendedBuildResult do_build_package(const VcpkgCmdArguments& args, const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) @@ -900,11 +1115,15 @@ namespace vcpkg::Build auto stdoutlog = buildpath / ("stdout-" + action.spec.triplet().canonical_name() + ".log"); int return_code; { + Sha512Scanner scanner{paths}; auto out_file = fs.open_for_write(stdoutlog, VCPKG_LINE_INFO); return_code = cmd_execute_and_stream_data( command, [&](StringView sv) { - print2(sv); + if (action.build_options.auto_update_mismatched_sha512 == Build::AutoUpdateMismatchedSHA512::YES) + scanner.process_input(sv); + else + print2(sv); Checks::check_exit(VCPKG_LINE_INFO, out_file.write(sv.data(), 1, sv.size()) == sv.size(), "Error occurred while writing '%s'", diff --git a/src/vcpkg/commands.xdownload.cpp b/src/vcpkg/commands.xdownload.cpp index 3ecc6827d0..302a5d8be0 100644 --- a/src/vcpkg/commands.xdownload.cpp +++ b/src/vcpkg/commands.xdownload.cpp @@ -14,12 +14,18 @@ namespace vcpkg::Commands::X_Download static constexpr StringLiteral OPTION_STORE = "store"; static constexpr StringLiteral OPTION_SKIP_SHA512 = "skip-sha512"; static constexpr StringLiteral OPTION_SHA512 = "sha512"; + static constexpr StringLiteral OPTION_ONLY_WARN_FOR_SHA512_MISMATCH = "only-warn-sha512-mismatch"; + static constexpr StringLiteral OPTION_SHA512_MISMATCH_GUID_WRAPPED = "sha512-mismatch-guid-wrapped"; static constexpr StringLiteral OPTION_URL = "url"; static constexpr StringLiteral OPTION_HEADER = "header"; static constexpr CommandSwitch FETCH_SWITCHES[] = { {OPTION_STORE, "Indicates the file should be stored instead of fetched"}, {OPTION_SKIP_SHA512, "Do not check the SHA512 of the downloaded file"}, + {OPTION_ONLY_WARN_FOR_SHA512_MISMATCH, + "Only warn if the SHA512 of the downloaded file was incorrect, instead of failing"}, + {OPTION_SHA512_MISMATCH_GUID_WRAPPED, + "Instead of displaying a user friendly message for a hash mismatch, print a machine readable one"}, }; static constexpr CommandSetting FETCH_SETTINGS[] = { {OPTION_SHA512, "The hash of the file to be downloaded"}, @@ -99,6 +105,13 @@ namespace vcpkg::Commands::X_Download auto file = fs.absolute(args.command_arguments[0], VCPKG_LINE_INFO); auto sha = get_sha512_check(args, parsed); + using Action = Downloads::Sha512MismatchAction; + auto mismatch_action = + Util::Sets::contains(parsed.switches, OPTION_ONLY_WARN_FOR_SHA512_MISMATCH) ? Action::Warn : Action::Error; + using Format = Downloads::Sha512MismatchFormat; + auto mismatch_format = Util::Sets::contains(parsed.switches, OPTION_SHA512_MISMATCH_GUID_WRAPPED) + ? Format::GuidWrapped + : Format::UserFriendly; // Is this a store command? if (Util::Sets::contains(parsed.switches, OPTION_STORE)) @@ -139,7 +152,7 @@ namespace vcpkg::Commands::X_Download urls = it_urls->second; } - download_manager.download_file(fs, urls, headers, file, sha); + download_manager.download_file(fs, urls, headers, file, sha, mismatch_action, mismatch_format); Checks::exit_success(VCPKG_LINE_INFO); } } diff --git a/src/vcpkg/install.cpp b/src/vcpkg/install.cpp index 10fc0008ce..bcd3177aaf 100644 --- a/src/vcpkg/install.cpp +++ b/src/vcpkg/install.cpp @@ -525,8 +525,9 @@ namespace vcpkg::Install static constexpr StringLiteral OPTION_PROHIBIT_BACKCOMPAT_FEATURES = "x-prohibit-backcompat-features"; static constexpr StringLiteral OPTION_ENFORCE_PORT_CHECKS = "enforce-port-checks"; static constexpr StringLiteral OPTION_ALLOW_UNSUPPORTED_PORT = "allow-unsupported"; + static constexpr StringLiteral OPTION_AUTO_UPDATE_SHA512_HASHES = "x-auto-update-sha512-hashes"; - static constexpr std::array INSTALL_SWITCHES = {{ + static constexpr std::array INSTALL_SWITCHES = {{ {OPTION_DRY_RUN, "Do not actually build or install"}, {OPTION_USE_HEAD_VERSION, "Install the libraries on the command line using the latest upstream sources"}, {OPTION_NO_DOWNLOADS, "Do not download new sources"}, @@ -545,8 +546,9 @@ namespace vcpkg::Install "Fail install if a port has detected problems or attempts to use a deprecated feature"}, {OPTION_PROHIBIT_BACKCOMPAT_FEATURES, ""}, {OPTION_ALLOW_UNSUPPORTED_PORT, "Instead of erroring on an unsupported port, continue with a warning."}, + {OPTION_AUTO_UPDATE_SHA512_HASHES, "(experimental) Auto updates wrong sha512 file hashes in the portfiles"}, }}; - static constexpr std::array MANIFEST_INSTALL_SWITCHES = {{ + static constexpr std::array MANIFEST_INSTALL_SWITCHES = {{ {OPTION_DRY_RUN, "Do not actually build or install"}, {OPTION_USE_HEAD_VERSION, "Install the libraries on the command line using the latest upstream sources"}, {OPTION_NO_DOWNLOADS, "Do not download new sources"}, @@ -565,6 +567,7 @@ namespace vcpkg::Install "Fail install if a port has detected problems or attempts to use a deprecated feature"}, {OPTION_PROHIBIT_BACKCOMPAT_FEATURES, ""}, {OPTION_ALLOW_UNSUPPORTED_PORT, "Instead of erroring on an unsupported port, continue with a warning."}, + {OPTION_AUTO_UPDATE_SHA512_HASHES, "(experimental) Auto updates wrong sha512 file hashes in the portfiles"}, }}; static constexpr std::array INSTALL_SETTINGS = {{ @@ -807,6 +810,7 @@ namespace vcpkg::Install const auto unsupported_port_action = Util::Sets::contains(options.switches, OPTION_ALLOW_UNSUPPORTED_PORT) ? Dependencies::UnsupportedPortAction::Warn : Dependencies::UnsupportedPortAction::Error; + const bool auto_update_512_hashes = Util::Sets::contains(options.switches, (OPTION_AUTO_UPDATE_SHA512_HASHES)); BinaryCache binary_cache; if (!only_downloads) @@ -831,6 +835,7 @@ namespace vcpkg::Install Build::PurgeDecompressFailure::NO, Util::Enum::to_enum(is_editable), prohibit_backcompat_features ? Build::BackcompatFeatures::PROHIBIT : Build::BackcompatFeatures::ALLOW, + Util::Enum::to_enum(auto_update_512_hashes), }; auto var_provider_storage = CMakeVars::make_triplet_cmake_var_provider(paths);