diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 945fe183404..738db132d16 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -161,7 +161,7 @@ void HttpBinaryCacheStore::upsertFile( } } -FileTransferRequest HttpBinaryCacheStore::makeRequest(const std::string & path) +FileTransferRequest HttpBinaryCacheStore::makeRequest(std::string_view path) { /* Otherwise the last path fragment will get discarded. */ auto cacheUriWithTrailingSlash = config->cacheUri; diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 402ee490082..305c33af142 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -155,9 +155,10 @@ struct FileTransferRequest unreachable(); } + void setupForS3(); + private: friend struct curlFileTransfer; - void setupForS3(); #if NIX_WITH_AWS_AUTH std::optional awsSigV4Provider; #endif diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index d8ba72390a4..ecad0997555 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -86,7 +86,7 @@ protected: const std::string & mimeType, uint64_t sizeHint) override; - FileTransferRequest makeRequest(const std::string & path); + FileTransferRequest makeRequest(std::string_view path); void getFile(const std::string & path, Sink & sink) override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 5d97fb0fdbd..3a5298e717a 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -4,6 +4,8 @@ #include #include +#include +#include namespace nix { @@ -26,6 +28,19 @@ class S3BinaryCacheStore : public virtual HttpBinaryCacheStore private: ref s3Config; + + struct UploadedPart + { + uint64_t partNumber; + std::string etag; + }; + + std::string createMultipartUpload( + const std::string_view key, + const std::string_view mimeType, + const std::optional contentEncoding); + void completeMultipartUpload( + const std::string_view key, const std::string_view uploadId, std::span parts); }; void S3BinaryCacheStore::upsertFile( @@ -37,6 +52,73 @@ void S3BinaryCacheStore::upsertFile( HttpBinaryCacheStore::upsertFile(path, istream, mimeType, sizeHint); } +// Creates a multipart upload for large objects to S3. +// See: +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html#API_CreateMultipartUpload_RequestSyntax +std::string S3BinaryCacheStore::createMultipartUpload( + const std::string_view key, const std::string_view mimeType, const std::optional contentEncoding) +{ + auto req = makeRequest(key); + + // setupForS3() converts s3:// to https:// but strips query parameters + // So we call it first, then add our multipart parameters + req.setupForS3(); + + auto url = req.uri.parsed(); + url.query["uploads"] = ""; + req.uri = VerbatimURL(url); + + req.method = HttpMethod::POST; + req.data = ""; + req.mimeType = mimeType; + + if (contentEncoding) { + req.headers.emplace_back("Content-Encoding", *contentEncoding); + } + + auto result = getFileTransfer()->enqueueFileTransfer(req).get(); + + std::regex uploadIdRegex("([^<]+)"); + std::smatch match; + + if (std::regex_search(result.data, match, uploadIdRegex)) { + return match[1]; + } + + throw Error("S3 CreateMultipartUpload response missing "); +} + +// Completes a multipart upload by combining all uploaded parts. +// See: +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html#API_CompleteMultipartUpload_RequestSyntax +void S3BinaryCacheStore::completeMultipartUpload( + const std::string_view key, const std::string_view uploadId, std::span parts) +{ + auto req = makeRequest(key); + req.setupForS3(); + + auto url = req.uri.parsed(); + url.query["uploadId"] = uploadId; + req.uri = VerbatimURL(url); + req.method = HttpMethod::POST; + + std::string xml = ""; + for (const auto & part : parts) { + xml += ""; + xml += "" + std::to_string(part.partNumber) + ""; + xml += "" + part.etag + ""; + xml += ""; + } + xml += ""; + + debug("S3 CompleteMultipartUpload XML (%d parts): %s", parts.size(), xml); + + req.data = xml; + req.mimeType = "text/xml"; + + getFileTransfer()->enqueueFileTransfer(req).get(); +} + StringSet S3BinaryCacheStoreConfig::uriSchemes() { return {"s3"};