From d768feaef30b053d7c60deba07044bc302489ac3 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sat, 25 Oct 2025 01:00:10 +0000 Subject: [PATCH 1/4] refactor(libstore): use string_view in HttpBinaryCacheStore::makeRequest --- src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/include/nix/store/http-binary-cache-store.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 945fe183404..738db132d16 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -161,7 +161,7 @@ void HttpBinaryCacheStore::upsertFile( } } -FileTransferRequest HttpBinaryCacheStore::makeRequest(const std::string & path) +FileTransferRequest HttpBinaryCacheStore::makeRequest(std::string_view path) { /* Otherwise the last path fragment will get discarded. */ auto cacheUriWithTrailingSlash = config->cacheUri; diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index d8ba72390a4..ecad0997555 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -86,7 +86,7 @@ protected: const std::string & mimeType, uint64_t sizeHint) override; - FileTransferRequest makeRequest(const std::string & path); + FileTransferRequest makeRequest(std::string_view path); void getFile(const std::string & path, Sink & sink) override; From ea2bdb81b1b5ec2b118a21d9b7a4a28562ecd297 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Fri, 24 Oct 2025 23:52:32 +0000 Subject: [PATCH 2/4] refactor(libstore/filetransfer): make setupForS3 public --- src/libstore/include/nix/store/filetransfer.hh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 402ee490082..305c33af142 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -155,9 +155,10 @@ struct FileTransferRequest unreachable(); } + void setupForS3(); + private: friend struct curlFileTransfer; - void setupForS3(); #if NIX_WITH_AWS_AUTH std::optional awsSigV4Provider; #endif From e421bdb8bbe41055a973412aa1358599d378bd64 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Fri, 24 Oct 2025 23:53:39 +0000 Subject: [PATCH 3/4] feat(libstore/s3-binary-cache-store): implement `createMultipartUpload()` POST to key with `?uploads` query parameter, optionally set `Content-Encoding` header, parse `uploadId` from XML response using regex --- src/libstore/s3-binary-cache-store.cc | 42 +++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 5d97fb0fdbd..2e05873e684 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -4,6 +4,7 @@ #include #include +#include namespace nix { @@ -26,6 +27,11 @@ class S3BinaryCacheStore : public virtual HttpBinaryCacheStore private: ref s3Config; + + std::string createMultipartUpload( + const std::string_view key, + const std::string_view mimeType, + const std::optional contentEncoding); }; void S3BinaryCacheStore::upsertFile( @@ -37,6 +43,42 @@ void S3BinaryCacheStore::upsertFile( HttpBinaryCacheStore::upsertFile(path, istream, mimeType, sizeHint); } +// Creates a multipart upload for large objects to S3. +// See: +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html#API_CreateMultipartUpload_RequestSyntax +std::string S3BinaryCacheStore::createMultipartUpload( + const std::string_view key, const std::string_view mimeType, const std::optional contentEncoding) +{ + auto req = makeRequest(key); + + // setupForS3() converts s3:// to https:// but strips query parameters + // So we call it first, then add our multipart parameters + req.setupForS3(); + + auto url = req.uri.parsed(); + url.query["uploads"] = ""; + req.uri = VerbatimURL(url); + + req.method = HttpMethod::POST; + req.data = ""; + req.mimeType = mimeType; + + if (contentEncoding) { + req.headers.emplace_back("Content-Encoding", *contentEncoding); + } + + auto result = getFileTransfer()->enqueueFileTransfer(req).get(); + + std::regex uploadIdRegex("([^<]+)"); + std::smatch match; + + if (std::regex_search(result.data, match, uploadIdRegex)) { + return match[1]; + } + + throw Error("S3 CreateMultipartUpload response missing "); +} + StringSet S3BinaryCacheStoreConfig::uriSchemes() { return {"s3"}; From c9cfa416bd34395791eaec617ce9451bab4ef55f Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Fri, 24 Oct 2025 23:54:49 +0000 Subject: [PATCH 4/4] feat(libstore/s3-binary-cache-store): implement `completeMultipartUpload()` `completeMultipartUpload()`: Build XML with part numbers and `ETags`, POST to key with `?uploadId` to finalize the multipart upload --- src/libstore/s3-binary-cache-store.cc | 40 +++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 2e05873e684..3a5298e717a 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -5,6 +5,7 @@ #include #include #include +#include namespace nix { @@ -28,10 +29,18 @@ class S3BinaryCacheStore : public virtual HttpBinaryCacheStore private: ref s3Config; + struct UploadedPart + { + uint64_t partNumber; + std::string etag; + }; + std::string createMultipartUpload( const std::string_view key, const std::string_view mimeType, const std::optional contentEncoding); + void completeMultipartUpload( + const std::string_view key, const std::string_view uploadId, std::span parts); }; void S3BinaryCacheStore::upsertFile( @@ -79,6 +88,37 @@ std::string S3BinaryCacheStore::createMultipartUpload( throw Error("S3 CreateMultipartUpload response missing "); } +// Completes a multipart upload by combining all uploaded parts. +// See: +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html#API_CompleteMultipartUpload_RequestSyntax +void S3BinaryCacheStore::completeMultipartUpload( + const std::string_view key, const std::string_view uploadId, std::span parts) +{ + auto req = makeRequest(key); + req.setupForS3(); + + auto url = req.uri.parsed(); + url.query["uploadId"] = uploadId; + req.uri = VerbatimURL(url); + req.method = HttpMethod::POST; + + std::string xml = ""; + for (const auto & part : parts) { + xml += ""; + xml += "" + std::to_string(part.partNumber) + ""; + xml += "" + part.etag + ""; + xml += ""; + } + xml += ""; + + debug("S3 CompleteMultipartUpload XML (%d parts): %s", parts.size(), xml); + + req.data = xml; + req.mimeType = "text/xml"; + + getFileTransfer()->enqueueFileTransfer(req).get(); +} + StringSet S3BinaryCacheStoreConfig::uriSchemes() { return {"s3"};