Skip to content

vcpkg install Auto update sha512 #166

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
autoantwort wants to merge 5 commits into from
Closed

vcpkg install Auto update sha512 #166

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6ff2bfd
x-download: Add option --only-warn-sha512-mismatch that cause the onl…
autoantwort Aug 21, 2021
bf26848
install: add option x-auto-update-sha512-hashes that updates wrong sh…
autoantwort Aug 21, 2021
8dd9b02
install: Use new guid based approach
autoantwort Sep 9, 2021
9282cfe
Apply suggestions from code review
autoantwort Sep 28, 2021
a5e3913
Merge branch 'main' into auto-update-sha512
autoantwort Nov 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 30 additions & 5 deletions include/vcpkg/base/downloads.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,17 @@ namespace vcpkg::Downloads
ExpectedS<SplitURIView> split_uri_view(StringView uri);
}

enum class Sha512MismatchFormat
{
UserFriendly,
GuidWrapped,
};

void verify_downloaded_file_hash(const Filesystem& fs,
const std::string& sanitized_url,
const Path& downloaded_path,
const std::string& sha512);
const std::string& sha512,
Sha512MismatchFormat mismatch_format = Sha512MismatchFormat::UserFriendly);

View<std::string> azure_blob_headers();

Expand All @@ -45,6 +52,18 @@ namespace vcpkg::Downloads
bool m_block_origin = false;
};

enum class Sha512MismatchAction : bool
{
Warn,
Error,
};

static constexpr StringLiteral guid_marker_hash_mismatch_start = "7279eda6-681f-46e0-aa5d-679ec14a2fb9";
static constexpr StringLiteral guid_marker_hash_mismatch_end = "6982135f-5ad4-406f-86e3-f2e19c8966ef";
// The following guids are used to detect the start and end of the output of the download command
static constexpr StringLiteral guid_marker_hash_mismatch_general_start = "b360a6a9-fb74-41de-a4c5-a7faf126d565";
static constexpr StringLiteral guid_marker_hash_mismatch_general_end = "9d36a06a-0efa-470a-9a1e-63a26be67a84";

// Handles downloading and uploading to a content addressable mirror
struct DownloadManager
{
Expand All @@ -55,23 +74,29 @@ namespace vcpkg::Downloads
void download_file(Filesystem& fs,
const std::string& url,
const Path& download_path,
const Optional<std::string>& sha512) const
const Optional<std::string>& sha512,
Sha512MismatchAction mismatch_action = Sha512MismatchAction::Error,
Sha512MismatchFormat mismatch_format = Sha512MismatchFormat::UserFriendly) const
{
this->download_file(fs, url, {}, download_path, sha512);
this->download_file(fs, url, {}, download_path, sha512, mismatch_action, mismatch_format);
}

void download_file(Filesystem& fs,
const std::string& url,
View<std::string> headers,
const Path& download_path,
const Optional<std::string>& sha512) const;
const Optional<std::string>& sha512,
Sha512MismatchAction mismatch_action = Sha512MismatchAction::Error,
Sha512MismatchFormat mismatch_format = Sha512MismatchFormat::UserFriendly) const;

// Returns url that was successfully downloaded from
std::string download_file(Filesystem& fs,
View<std::string> urls,
View<std::string> headers,
const Path& download_path,
const Optional<std::string>& sha512) const;
const Optional<std::string>& sha512,
Sha512MismatchAction mismatch_action = Sha512MismatchAction::Error,
Sha512MismatchFormat mismatch_format = Sha512MismatchFormat::UserFriendly) const;

ExpectedS<int> put_file_to_mirror(const Filesystem& fs, const Path& file_to_put, StringView sha512) const;

Expand Down
7 changes: 7 additions & 0 deletions include/vcpkg/build.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,12 @@ namespace vcpkg::Build
YES
};

enum class AutoUpdateMismatchedSHA512 : bool
{
NO,
YES
};

struct BuildPackageOptions
{
BuildMissing build_missing;
Expand All @@ -165,6 +171,7 @@ namespace vcpkg::Build
PurgeDecompressFailure purge_decompress_failure;
Editable editable;
BackcompatFeatures backcompat_features;
AutoUpdateMismatchedSHA512 auto_update_mismatched_sha512;
};

static constexpr BuildPackageOptions default_build_package_options{
Expand Down
97 changes: 73 additions & 24 deletions src/vcpkg/base/downloads.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,23 +189,36 @@ namespace vcpkg::Downloads
static std::string format_hash_mismatch(StringView url,
const Path& downloaded_path,
StringView expected,
StringView actual)
StringView actual,
Sha512MismatchFormat mismatch_format)
{
return Strings::format("File does not have the expected hash:\n"
" url : [ %s ]\n"
" File path : [ %s ]\n"
" Expected hash : [ %s ]\n"
" Actual hash : [ %s ]\n",
url,
downloaded_path,
if (mismatch_format == Sha512MismatchFormat::UserFriendly)
{
return Strings::format("File does not have the expected hash:\n"
" url : [ %s ]\n"
" File path : [ %s ]\n"
" Expected hash : [ %s ]\n"
" Actual hash : [ %s ]\n",
url,
downloaded_path,
expected,
actual);
}
return Strings::format("%s\n"
"expected=%s\n"
"actual=%s\n"
"%s\n",
guid_marker_hash_mismatch_start,
expected,
actual);
actual,
guid_marker_hash_mismatch_end);
}

static Optional<std::string> try_verify_downloaded_file_hash(const Filesystem& fs,
StringView sanitized_url,
const Path& downloaded_path,
StringView sha512)
StringView sha512,
Sha512MismatchFormat mismatch_format)
{
std::string actual_hash =
vcpkg::Hash::get_file_hash(VCPKG_LINE_INFO, fs, downloaded_path, Hash::Algorithm::Sha512);
Expand All @@ -223,17 +236,18 @@ namespace vcpkg::Downloads

if (sha512 != actual_hash)
{
return format_hash_mismatch(sanitized_url, downloaded_path, sha512, actual_hash);
return format_hash_mismatch(sanitized_url, downloaded_path, sha512, actual_hash, mismatch_format);
}
return nullopt;
}

void verify_downloaded_file_hash(const Filesystem& fs,
const std::string& url,
const Path& downloaded_path,
const std::string& sha512)
const std::string& sha512,
Sha512MismatchFormat mismatch_format)
{
auto maybe_error = try_verify_downloaded_file_hash(fs, url, downloaded_path, sha512);
auto maybe_error = try_verify_downloaded_file_hash(fs, url, downloaded_path, sha512, mismatch_format);
if (auto err = maybe_error.get())
{
Checks::exit_with_message(VCPKG_LINE_INFO, *err);
Expand All @@ -242,15 +256,23 @@ namespace vcpkg::Downloads

static bool check_downloaded_file_hash(Filesystem& fs,
const Optional<std::string>& hash,
Sha512MismatchAction mismatch_action,
Sha512MismatchFormat mismatch_format,
StringView sanitized_url,
const Path& download_part_path,
std::string& errors)
{
if (auto p = hash.get())
{
auto maybe_error = try_verify_downloaded_file_hash(fs, sanitized_url, download_part_path, *p);
auto maybe_error =
try_verify_downloaded_file_hash(fs, sanitized_url, download_part_path, *p, mismatch_format);
if (auto err = maybe_error.get())
{
if (mismatch_action == Sha512MismatchAction::Warn)
{
print2(Color::warning, *err, '\n');
return true;
}
Strings::append(errors, *err, '\n');
return false;
}
Expand Down Expand Up @@ -511,6 +533,8 @@ namespace vcpkg::Downloads
View<std::string> headers,
const Path& download_path,
const Optional<std::string>& sha512,
Sha512MismatchAction mismatch_action,
Sha512MismatchFormat mismatch_format,
const std::vector<std::string>& secrets,
std::string& errors)
{
Expand All @@ -535,7 +559,8 @@ namespace vcpkg::Downloads
{
if (download_winhttp(fs, download_path_part_path, split_uri, url, secrets, errors))
{
if (check_downloaded_file_hash(fs, sha512, url, download_path_part_path, errors))
if (check_downloaded_file_hash(
fs, sha512, mismatch_action, mismatch_format, url, download_path_part_path, errors))
{
fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO);
return true;
Expand Down Expand Up @@ -566,7 +591,8 @@ namespace vcpkg::Downloads
return false;
}

if (check_downloaded_file_hash(fs, sha512, sanitized_url, download_path_part_path, errors))
if (check_downloaded_file_hash(
fs, sha512, mismatch_action, mismatch_format, sanitized_url, download_path_part_path, errors))
{
fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO);
return true;
Expand All @@ -579,12 +605,16 @@ namespace vcpkg::Downloads
View<std::string> headers,
const Path& download_path,
const Optional<std::string>& sha512,
Sha512MismatchAction mismatch_action,
Sha512MismatchFormat mismatch_format,
const std::vector<std::string>& secrets,
std::string& errors)
{
for (auto&& url : urls)
{
if (try_download_file(fs, url, headers, download_path, sha512, secrets, errors)) return url;
if (try_download_file(
fs, url, headers, download_path, sha512, mismatch_action, mismatch_format, secrets, errors))
return url;
}
return nullopt;
}
Expand All @@ -599,25 +629,37 @@ namespace vcpkg::Downloads
const std::string& url,
View<std::string> headers,
const Path& download_path,
const Optional<std::string>& sha512) const
const Optional<std::string>& sha512,
Sha512MismatchAction mismatch_action,
Sha512MismatchFormat mismatch_format) const
{
this->download_file(fs, View<std::string>(&url, 1), headers, download_path, sha512);
this->download_file(
fs, View<std::string>(&url, 1), headers, download_path, sha512, mismatch_action, mismatch_format);
}

std::string DownloadManager::download_file(Filesystem& fs,
View<std::string> urls,
View<std::string> headers,
const Path& download_path,
const Optional<std::string>& sha512) const
const Optional<std::string>& sha512,
Sha512MismatchAction mismatch_action,
Sha512MismatchFormat mismatch_format) const
{
std::string errors;
if (auto hash = sha512.get())
{
if (auto read_template = m_config.m_read_url_template.get())
{
auto read_url = Strings::replace_all(*read_template, "<SHA>", *hash);
if (Downloads::try_download_file(
fs, read_url, m_config.m_read_headers, download_path, sha512, m_config.m_secrets, errors))
if (Downloads::try_download_file(fs,
read_url,
m_config.m_read_headers,
download_path,
sha512,
mismatch_action,
mismatch_format,
m_config.m_secrets,
errors))
return read_url;
}
}
Expand All @@ -637,8 +679,15 @@ namespace vcpkg::Downloads
}
else
{
auto maybe_url =
try_download_files(fs, urls, headers, download_path, sha512, m_config.m_secrets, errors);
auto maybe_url = try_download_files(fs,
urls,
headers,
download_path,
sha512,
mismatch_action,
mismatch_format,
m_config.m_secrets,
errors);
if (auto url = maybe_url.get())
{
if (auto hash = sha512.get())
Expand Down
Loading