feat(libstore/s3-binary-cache-store): implement create/completeMultipartUpload()

src/libstore/http-binary-cache-store.cc

@@ -161,7 +161,7 @@ void HttpBinaryCacheStore::upsertFile(
     }
 }
 
-FileTransferRequest HttpBinaryCacheStore::makeRequest(const std::string & path)
+FileTransferRequest HttpBinaryCacheStore::makeRequest(std::string_view path)
 {
     /* Otherwise the last path fragment will get discarded. */
     auto cacheUriWithTrailingSlash = config->cacheUri;

src/libstore/include/nix/store/filetransfer.hh

@@ -155,9 +155,10 @@ struct FileTransferRequest
         unreachable();
     }
 
+    void setupForS3();
+
 private:
     friend struct curlFileTransfer;
-    void setupForS3();
 #if NIX_WITH_AWS_AUTH
     std::optional<std::string> awsSigV4Provider;
 #endif

src/libstore/include/nix/store/http-binary-cache-store.hh

@@ -86,7 +86,7 @@ protected:
         const std::string & mimeType,
         uint64_t sizeHint) override;
 
-    FileTransferRequest makeRequest(const std::string & path);
+    FileTransferRequest makeRequest(std::string_view path);
 
     void getFile(const std::string & path, Sink & sink) override;
 

src/libstore/s3-binary-cache-store.cc

@@ -4,6 +4,8 @@
 
 #include <cassert>
 #include <ranges>
+#include <regex>
+#include <span>
 
 namespace nix {
 
@@ -26,6 +28,19 @@ class S3BinaryCacheStore : public virtual HttpBinaryCacheStore
 
 private:
     ref<S3BinaryCacheStoreConfig> s3Config;
+
+    struct UploadedPart
+    {
+        uint64_t partNumber;
+        std::string etag;
+    };
+
+    std::string createMultipartUpload(
+        const std::string_view key,
+        const std::string_view mimeType,
+        const std::optional<std::string_view> contentEncoding);
+    void completeMultipartUpload(
+        const std::string_view key, const std::string_view uploadId, std::span<const UploadedPart> parts);
 };
 
 void S3BinaryCacheStore::upsertFile(
@@ -37,6 +52,73 @@ void S3BinaryCacheStore::upsertFile(
     HttpBinaryCacheStore::upsertFile(path, istream, mimeType, sizeHint);
 }
 
+// Creates a multipart upload for large objects to S3.
+// See:
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html#API_CreateMultipartUpload_RequestSyntax
+std::string S3BinaryCacheStore::createMultipartUpload(
+    const std::string_view key, const std::string_view mimeType, const std::optional<std::string_view> contentEncoding)
+{
+    auto req = makeRequest(key);
+
+    // setupForS3() converts s3:// to https:// but strips query parameters
+    // So we call it first, then add our multipart parameters
+    req.setupForS3();
+
+    auto url = req.uri.parsed();
+    url.query["uploads"] = "";
+    req.uri = VerbatimURL(url);
+
+    req.method = HttpMethod::POST;
+    req.data = "";
+    req.mimeType = mimeType;
+
+    if (contentEncoding) {
+        req.headers.emplace_back("Content-Encoding", *contentEncoding);
+    }
+
+    auto result = getFileTransfer()->enqueueFileTransfer(req).get();
+
+    std::regex uploadIdRegex("<UploadId>([^<]+)</UploadId>");
+    std::smatch match;
+
+    if (std::regex_search(result.data, match, uploadIdRegex)) {
+        return match[1];
+    }
+
+    throw Error("S3 CreateMultipartUpload response missing <UploadId>");
+}
+
+// Completes a multipart upload by combining all uploaded parts.
+// See:
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html#API_CompleteMultipartUpload_RequestSyntax
+void S3BinaryCacheStore::completeMultipartUpload(
+    const std::string_view key, const std::string_view uploadId, std::span<const UploadedPart> parts)
+{
+    auto req = makeRequest(key);
+    req.setupForS3();
+
+    auto url = req.uri.parsed();
+    url.query["uploadId"] = uploadId;
+    req.uri = VerbatimURL(url);
+    req.method = HttpMethod::POST;
+
+    std::string xml = "<CompleteMultipartUpload>";
+    for (const auto & part : parts) {
+        xml += "<Part>";
+        xml += "<PartNumber>" + std::to_string(part.partNumber) + "</PartNumber>";
+        xml += "<ETag>" + part.etag + "</ETag>";
+        xml += "</Part>";
+    }
+    xml += "</CompleteMultipartUpload>";
+
+    debug("S3 CompleteMultipartUpload XML (%d parts): %s", parts.size(), xml);
+
+    req.data = xml;
+    req.mimeType = "text/xml";
+
+    getFileTransfer()->enqueueFileTransfer(req).get();
+}
+
 StringSet S3BinaryCacheStoreConfig::uriSchemes()
 {
     return {"s3"};