Skip to content

[vcpkg] Move default binary cache from $root/archives to user-wide directory #12256

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ras0219-msft merged 3 commits into from
Jul 9, 2020
Merged

[vcpkg] Move default binary cache from $root/archives to user-wide directory #12256

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fa0df52
[vcpkg] Move default binary cache from `$root/archives` to user-wide …
ras0219-msft Jul 3, 2020
5110b5c
Merge remote-tracking branch 'origin/master' into dev/roschuma/binary…
ras0219-msft Jul 9, 2020
0ad6e8e
[vcpkg] Reformat
ras0219-msft Jul 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions scripts/azure-pipelines/end-to-end-tests.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ if (-not $IsLinux -and -not $IsMacOS)
Refresh-TestRoot

# Test simple installation
$args = $commonArgs + @("install","rapidjson","--binarycaching","--x-binarysource=clear;files,$ArchiveRoot,write;nuget,$NuGetRoot,upload")
$args = $commonArgs + @("install","rapidjson","--binarycaching","--x-binarysource=clear;files,$ArchiveRoot,write;nuget,$NuGetRoot,readwrite")
$CurrentTest = "./vcpkg $($args -join ' ')"
Write-Host $CurrentTest
./vcpkg @args
Expand Down Expand Up @@ -164,7 +164,7 @@ Throw-IfFailed
Remove-Item -Recurse -Force $NuGetRoot -ErrorAction SilentlyContinue
mkdir $NuGetRoot

$args = $commonArgs + @("install","rapidjson","tinyxml","--binarycaching","--x-binarysource=clear;nuget,$NuGetRoot2;nuget,$NuGetRoot,upload")
$args = $commonArgs + @("install","rapidjson","tinyxml","--binarycaching","--x-binarysource=clear;nuget,$NuGetRoot2;nuget,$NuGetRoot,write")
$CurrentTest = "./vcpkg $($args -join ' ')"
Write-Host $CurrentTest
./vcpkg @args
Expand Down
3 changes: 1 addition & 2 deletions toolsrc/include/vcpkg/binarycaching.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,7 @@ namespace vcpkg

IBinaryProvider& null_binary_provider();

ExpectedS<std::unique_ptr<IBinaryProvider>> create_binary_provider_from_configs(const VcpkgPaths& paths,
View<std::string> args);
ExpectedS<std::unique_ptr<IBinaryProvider>> create_binary_provider_from_configs(View<std::string> args);
ExpectedS<std::unique_ptr<IBinaryProvider>> create_binary_provider_from_configs_pure(const std::string& env_string,
View<std::string> args);

Expand Down
30 changes: 23 additions & 7 deletions toolsrc/src/vcpkg-test/binaryconfigparser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,15 +89,15 @@ TEST_CASE ("BinaryConfigParser nuget source provider", "[binaryconfigparser]")
REQUIRE(!parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("nuget," ABSOLUTE_PATH ",upload", {});
auto parsed = create_binary_provider_from_configs_pure("nuget," ABSOLUTE_PATH ",readwrite", {});
REQUIRE(parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("nuget," ABSOLUTE_PATH ",upload,extra", {});
auto parsed = create_binary_provider_from_configs_pure("nuget," ABSOLUTE_PATH ",readwrite,extra", {});
REQUIRE(!parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("nuget,,upload", {});
auto parsed = create_binary_provider_from_configs_pure("nuget,,readwrite", {});
REQUIRE(!parsed.has_value());
}
}
Expand Down Expand Up @@ -125,15 +125,23 @@ TEST_CASE ("BinaryConfigParser nuget config provider", "[binaryconfigparser]")
REQUIRE(!parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("nugetconfig," ABSOLUTE_PATH ",upload", {});
auto parsed = create_binary_provider_from_configs_pure("nugetconfig," ABSOLUTE_PATH ",read", {});
REQUIRE(parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("nugetconfig," ABSOLUTE_PATH ",upload,extra", {});
auto parsed = create_binary_provider_from_configs_pure("nugetconfig," ABSOLUTE_PATH ",write", {});
REQUIRE(parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("nugetconfig," ABSOLUTE_PATH ",readwrite", {});
REQUIRE(parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("nugetconfig," ABSOLUTE_PATH ",readwrite,extra", {});
REQUIRE(!parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("nugetconfig,,upload", {});
auto parsed = create_binary_provider_from_configs_pure("nugetconfig,,readwrite", {});
REQUIRE(!parsed.has_value());
}
}
Expand Down Expand Up @@ -185,7 +193,7 @@ TEST_CASE ("BinaryConfigParser interactive provider", "[binaryconfigparser]")
REQUIRE(parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("interactive,upload", {});
auto parsed = create_binary_provider_from_configs_pure("interactive,read", {});
REQUIRE(!parsed.has_value());
}
}
Expand All @@ -196,6 +204,14 @@ TEST_CASE ("BinaryConfigParser multiple providers", "[binaryconfigparser]")
auto parsed = create_binary_provider_from_configs_pure("clear;default", {});
REQUIRE(parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("clear;default,read", {});
REQUIRE(parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("clear;default,write", {});
REQUIRE(parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("clear;default,readwrite", {});
REQUIRE(parsed.has_value());
Expand Down
164 changes: 55 additions & 109 deletions toolsrc/src/vcpkg/binarycaching.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -651,23 +651,13 @@ IBinaryProvider& vcpkg::null_binary_provider()
return p;
}

ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_configs(const VcpkgPaths& paths,
View<std::string> args)
ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_configs(View<std::string> args)
{
std::string env_string = System::get_environment_variable("VCPKG_BINARY_SOURCES").value_or("");

// Preserve existing behavior until CI can be updated
// TODO: remove
if (args.size() == 0 && env_string.empty())
{
auto p = paths.root / fs::u8path("archives");
return {std::make_unique<ArchivesBinaryProvider>(std::vector<fs::path>{p}, std::vector<fs::path>{p})};
}

return create_binary_provider_from_configs_pure(env_string, args);
}
ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_configs_pure(
const std::string& env_string, View<std::string> args)
namespace
{
struct State
{
Expand Down Expand Up @@ -755,6 +745,41 @@ ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_c
}
}

template<class T>
void handle_readwrite(std::vector<T>& read,
std::vector<T>& write,
T&& t,
const std::vector<std::pair<SourceLoc, std::string>>& segments,
size_t segment_idx)
{
if (segment_idx >= segments.size())
{
read.push_back(std::move(t));
return;
}

auto& mode = segments[segment_idx].second;

if (mode == "read")
{
read.push_back(std::move(t));
}
else if (mode == "write")
{
write.push_back(std::move(t));
}
else if (mode == "readwrite")
{
read.push_back(t);
write.push_back(std::move(t));
}
else
{
return add_error("unexpected argument: expected 'read', readwrite', or 'write'",
segments[segment_idx].first);
}
}

void handle_segments(std::vector<std::pair<SourceLoc, std::string>>&& segments)
{
if (segments.empty()) return;
Expand All @@ -779,37 +804,10 @@ ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_c
return add_error("expected arguments: path arguments for binary config strings must be absolute",
segments[1].first);
}

std::string mode;
switch (segments.size())
{
case 2: mode = "read"; break;
case 3: mode = segments[2].second; break;
default:
return add_error("unexpected arguments: binary config 'files' requires 1 or 2 arguments",
segments[3].first);
}

if (mode == "read")
{
state->archives_to_read.push_back(std::move(p));
}
else if (mode == "write")
{
state->archives_to_write.push_back(std::move(p));
}
else if (mode == "readwrite")
{
state->archives_to_read.push_back(p);
state->archives_to_write.push_back(std::move(p));
}
else
{
Checks::check_exit(VCPKG_LINE_INFO, segments.size() > 2);
return add_error("unexpected arguments: binary config 'files' can only accept"
" 'read', readwrite', or 'write' as a second argument",
segments[2].first);
}
handle_readwrite(state->archives_to_read, state->archives_to_write, std::move(p), segments, 2);
if (segments.size() > 3)
return add_error("unexpected arguments: binary config 'files' requires 1 or 2 arguments",
segments[3].first);
}
else if (segments[0].second == "interactive")
{
Expand All @@ -829,28 +827,10 @@ ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_c
if (!p.is_absolute())
return add_error("expected arguments: path arguments for binary config strings must be absolute",
segments[1].first);

handle_readwrite(state->configs_to_read, state->configs_to_write, std::move(p), segments, 2);
if (segments.size() > 3)
{
return add_error(
"unexpected arguments: binary config 'nugetconfig' does not take more than 2 arguments",
segments[3].first);
}
else if (segments.size() == 3)
{
if (segments[2].second != "upload")
{
return add_error(
"unexpected arguments: binary config 'nugetconfig' can only accept 'upload' as "
"a second argument",
segments[2].first);
}
else
{
state->configs_to_write.push_back(p);
}
}
state->configs_to_read.push_back(std::move(p));
return add_error("unexpected arguments: binary config 'nugetconfig' requires 1 or 2 arguments",
segments[3].first);
}
else if (segments[0].second == "nuget")
{
Expand All @@ -861,25 +841,11 @@ ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_c
auto&& p = segments[1].second;
if (p.empty())
return add_error("unexpected arguments: binary config 'nuget' requires non-empty source");

handle_readwrite(state->sources_to_read, state->sources_to_write, std::move(p), segments, 2);
if (segments.size() > 3)
{
return add_error("unexpected arguments: binary config 'nuget' does not take more than 2 arguments",
return add_error("unexpected arguments: binary config 'nuget' requires 1 or 2 arguments",
segments[3].first);
}
else if (segments.size() == 3)
{
if (segments[2].second != "upload")
{
return add_error("unexpected arguments: binary config 'nuget' can only accept 'upload' as "
"a second argument",
segments[2].first);
}
else
{
state->sources_to_write.push_back(p);
}
}
state->sources_to_read.push_back(std::move(p));
}
else if (segments[0].second == "default")
{
Expand All @@ -899,34 +865,7 @@ ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_c
return add_error("default path was not absolute: " + p.u8string(), segments[0].first);
}

std::string mode;
switch (segments.size())
{
case 1: mode = "read"; break;
case 2: mode = segments[1].second; break;
default: Checks::unreachable(VCPKG_LINE_INFO);
}

if (mode == "read")
{
state->archives_to_read.push_back(std::move(p));
}
else if (mode == "write")
{
state->archives_to_write.push_back(std::move(p));
}
else if (mode == "readwrite")
{
state->archives_to_read.push_back(p);
state->archives_to_write.push_back(std::move(p));
}
else
{
Checks::check_exit(VCPKG_LINE_INFO, segments.size() > 1);
return add_error("unexpected arguments: binary config 'default' can only accept"
" 'read', readwrite', or 'write' as a first argument",
segments[1].first);
}
handle_readwrite(state->archives_to_read, state->archives_to_write, std::move(p), segments, 1);
}
else
{
Expand All @@ -937,9 +876,16 @@ ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_c
}
}
};
}

ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_configs_pure(
const std::string& env_string, View<std::string> args)
{
State s;

BinaryConfigParser default_parser("default,readwrite", "<defaults>", &s);
default_parser.parse();

BinaryConfigParser env_parser(env_string, "VCPKG_BINARY_SOURCES", &s);
env_parser.parse();
if (auto err = env_parser.get_error()) return err->format();
Expand Down
3 changes: 1 addition & 2 deletions toolsrc/src/vcpkg/build.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,8 +164,7 @@ namespace vcpkg::Build
const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE);
std::string first_arg = args.command_arguments.at(0);

auto binaryprovider =
create_binary_provider_from_configs(paths, args.binary_sources).value_or_exit(VCPKG_LINE_INFO);
auto binaryprovider = create_binary_provider_from_configs(args.binary_sources).value_or_exit(VCPKG_LINE_INFO);

const FullPackageSpec spec = Input::check_and_get_full_package_spec(
std::move(first_arg), default_triplet, COMMAND_STRUCTURE.example_text);
Expand Down
3 changes: 1 addition & 2 deletions toolsrc/src/vcpkg/commands.buildexternal.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,7 @@ namespace vcpkg::Commands::BuildExternal
{
const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE);

auto binaryprovider =
create_binary_provider_from_configs(paths, args.binary_sources).value_or_exit(VCPKG_LINE_INFO);
auto binaryprovider = create_binary_provider_from_configs(args.binary_sources).value_or_exit(VCPKG_LINE_INFO);

const FullPackageSpec spec = Input::check_and_get_full_package_spec(
std::string(args.command_arguments.at(0)), default_triplet, COMMAND_STRUCTURE.example_text);
Expand Down
2 changes: 1 addition & 1 deletion toolsrc/src/vcpkg/commands.ci.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@ namespace vcpkg::Commands::CI
if (args.binary_caching_enabled())
{
binaryproviderStorage =
create_binary_provider_from_configs(paths, args.binary_sources).value_or_exit(VCPKG_LINE_INFO);
create_binary_provider_from_configs(args.binary_sources).value_or_exit(VCPKG_LINE_INFO);
}

IBinaryProvider& binaryprovider = binaryproviderStorage ? *binaryproviderStorage : null_binary_provider();
Expand Down
3 changes: 1 addition & 2 deletions toolsrc/src/vcpkg/commands.setinstalled.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,7 @@ namespace vcpkg::Commands::SetInstalled
Input::check_triplet(spec.package_spec.triplet(), paths);
}

auto binary_provider =
create_binary_provider_from_configs(paths, args.binary_sources).value_or_exit(VCPKG_LINE_INFO);
auto binary_provider = create_binary_provider_from_configs(args.binary_sources).value_or_exit(VCPKG_LINE_INFO);

const bool dry_run = Util::Sets::contains(options.switches, OPTION_DRY_RUN);

Expand Down
3 changes: 1 addition & 2 deletions toolsrc/src/vcpkg/commands.upgrade.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,7 @@ namespace vcpkg::Commands::Upgrade
const bool no_dry_run = Util::Sets::contains(options.switches, OPTION_NO_DRY_RUN);
const KeepGoing keep_going = to_keep_going(Util::Sets::contains(options.switches, OPTION_KEEP_GOING));

auto binaryprovider =
create_binary_provider_from_configs(paths, args.binary_sources).value_or_exit(VCPKG_LINE_INFO);
auto binaryprovider = create_binary_provider_from_configs(args.binary_sources).value_or_exit(VCPKG_LINE_INFO);

StatusParagraphs status_db = database_load_check(paths);

Expand Down
3 changes: 1 addition & 2 deletions toolsrc/src/vcpkg/install.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -672,8 +672,7 @@ namespace vcpkg::Install
const ParsedArguments options =
args.parse_arguments(paths.manifest_mode_enabled() ? MANIFEST_COMMAND_STRUCTURE : COMMAND_STRUCTURE);

auto binaryprovider =
create_binary_provider_from_configs(paths, args.binary_sources).value_or_exit(VCPKG_LINE_INFO);
auto binaryprovider = create_binary_provider_from_configs(args.binary_sources).value_or_exit(VCPKG_LINE_INFO);

const bool dry_run = Util::Sets::contains(options.switches, OPTION_DRY_RUN);
const bool use_head_version = Util::Sets::contains(options.switches, (OPTION_USE_HEAD_VERSION));
Expand Down