Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions src/libstore/include/nix/store/s3-binary-cache-store.hh
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,37 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
> addressing instead of virtual host based addressing.
)"};

const Setting<bool> multipartUpload{
this,
false,
"multipart-upload",
R"(
Whether to use multipart uploads for large files. When enabled,
files exceeding the multipart threshold will be uploaded in
multiple parts, which is required for files larger than 5 GiB and
can improve performance and reliability for large uploads.
)"};

const Setting<uint64_t> multipartChunkSize{
this,
5 * 1024 * 1024,
"multipart-chunk-size",
R"(
The size (in bytes) of each part in multipart uploads. Must be
at least 5 MiB (AWS S3 requirement). Larger chunk sizes reduce the
number of requests but use more memory. Default is 5 MiB.
)"};

const Setting<uint64_t> multipartThreshold{
this,
100 * 1024 * 1024,
"multipart-threshold",
R"(
The minimum file size (in bytes) for using multipart uploads.
Files smaller than this threshold will use regular PUT requests.
Default is 100 MiB. Only takes effect when multipart-upload is enabled.
)"};

/**
* Set of settings that are part of the S3 URI itself.
* These are needed for region specification and other S3-specific settings.
Expand Down
27 changes: 27 additions & 0 deletions src/libstore/s3-binary-cache-store.cc
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
#include "nix/store/s3-binary-cache-store.hh"
#include "nix/store/http-binary-cache-store.hh"
#include "nix/store/store-registration.hh"
#include "nix/util/error.hh"
#include "nix/util/logging.hh"

#include <cassert>
#include <ranges>
Expand All @@ -27,6 +29,31 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig(
cacheUri.query[key] = value;
}
}

const uint64_t AWS_MIN_PART_SIZE = 5 * 1024 * 1024;
const uint64_t AWS_MAX_PART_SIZE = 5ULL * 1024 * 1024 * 1024;

if (multipartChunkSize < AWS_MIN_PART_SIZE) {
throw UsageError(
"multipart-chunk-size must be at least 5 MiB (5242880 bytes), got %d bytes (%d MiB)",
multipartChunkSize.get(),
multipartChunkSize.get() / (1024 * 1024));
}

if (multipartChunkSize > AWS_MAX_PART_SIZE) {
throw UsageError(
"multipart-chunk-size must be at most 5 GiB (5368709120 bytes), got %d bytes (%d GiB)",
multipartChunkSize.get(),
multipartChunkSize.get() / (1024 * 1024 * 1024));
}

if (multipartUpload && multipartThreshold < multipartChunkSize) {
warn(
"multipart-threshold (%d MiB) is less than multipart-chunk-size (%d MiB), "
"which may result in single-part multipart uploads",
multipartThreshold.get() / (1024 * 1024),
multipartChunkSize.get() / (1024 * 1024));
}
}

std::string S3BinaryCacheStoreConfig::getHumanReadableURI() const
Expand Down
Loading