diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e370e278c52..c3728e2b3c1 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -147,7 +147,7 @@ struct TunnelLogger : public Logger } }; -struct TunnelSink : Sink +struct TunnelSink : public Sink { Sink & to; TunnelSink(Sink & to) : to(to) { } @@ -175,7 +175,7 @@ struct TunnelSource : BufferedSource /* If the NAR archive contains a single file at top-level, then save the contents of the file to `s'. Otherwise barf. */ -struct RetrieveRegularNARSink : ParseSink +struct RetrieveRegularNARSink : public ParseSink { bool regular; string s; @@ -721,9 +721,10 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 21) source = std::make_unique(from, to); else { - TeeSink tee(from); - parseDump(tee, tee.source); - saved = std::move(*tee.source.data); + TeeSource tee(from); + ParseSink sink; + parseDump(sink, tee); + saved = std::move(*tee.data); source = std::make_unique(saved); } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index cb9da027dd6..38c21cb6c7b 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -7,7 +7,7 @@ namespace nix { -struct HashAndWriteSink : Sink +struct HashAndWriteSink : public Sink { Sink & writeSink; HashSink hashSink; @@ -77,8 +77,9 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr acces if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); /* Extract the NAR from the source. */ - TeeSink tee(source); - parseDump(tee, tee.source); + TeeSource tee(source); + ParseSink sink; + parseDump(sink, tee); uint32_t magic = readInt(source); if (magic != exportMagic) @@ -94,15 +95,15 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr acces if (deriver != "") info.deriver = parseStorePath(deriver); - info.narHash = hashString(htSHA256, *tee.source.data); - info.narSize = tee.source.data->size(); + info.narHash = hashString(htSHA256, *tee.data); + info.narSize = tee.data->size(); // Ignore optional legacy signature. if (readInt(source) == 1) readString(source); // Can't use underlying source, which would have been exhausted - auto source = StringSource { *tee.source.data }; + auto source = StringSource { *tee.data }; addToStore(info, source, NoRepair, checkSigs, accessor); res.push_back(info.path); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index c954ace7f77..43cebf01d7b 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -22,6 +22,7 @@ #include #include #include +#include using namespace std::string_literals; @@ -56,7 +57,6 @@ struct curlFileTransfer : public FileTransfer Callback callback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object - std::string status; unsigned int attempt = 0; @@ -72,6 +72,8 @@ struct curlFileTransfer : public FileTransfer curl_off_t writtenToSink = 0; + inline static const std::set successfulStatuses; + /* Get the HTTP status code, or 0 for other protocols. */ long getHTTPStatus() { @@ -98,7 +100,7 @@ struct curlFileTransfer : public FileTransfer /* Only write data to the sink if this is a successful response. */ - if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) { + if (successfulStatuses.find(httpStatus) != successfulStatuses.end()) { writtenToSink += len; this->request.dataCallback((char *) data, len); } @@ -122,7 +124,7 @@ struct curlFileTransfer : public FileTransfer if (requestHeaders) curl_slist_free_all(requestHeaders); try { if (!done) - fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri)); + fail(FileTransferError(Interrupted, nullptr, "download of '%s' was interrupted", request.uri)); } catch (...) { ignoreException(); } @@ -142,7 +144,9 @@ struct curlFileTransfer : public FileTransfer } LambdaSink finalSink; - std::shared_ptr decompressionSink; + // std::shared_ptr< std::variant > decompressionSink; + // std::variant< std::shared_ptr, std::shared_ptr > decompressionSink; + std::shared_ptr decompressionSink; std::exception_ptr writeException; @@ -152,8 +156,13 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; result.bodySize += realSize; - if (!decompressionSink) - decompressionSink = makeDecompressionSink(encoding, finalSink); + // if (!decompressionSink) + // // decompressionSink = makeDecompressionSink(encoding, finalSink); + // decompressionSink.reset(*makeDecompressionSink(encoding, finalSink)); + + // Let's see some types: + // makeDecompressionSink(...) :: ref + // (*decompressionSink)((unsigned char *) contents, realSize); @@ -174,6 +183,7 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; std::string line((char *) contents, realSize); printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); + std::string status; if (line.compare(0, 5, "HTTP/") == 0) { // new response starts result.etag = ""; auto ss = tokenizeString>(line, " "); @@ -182,6 +192,13 @@ struct curlFileTransfer : public FileTransfer result.bodySize = 0; acceptRanges = false; encoding = ""; + if (successfulStatuses.find(std::stoi(status)) != successfulStatuses.end()) { + // Here we want to construct just the sink + decompressionSink.reset(static_cast(makeDecompressionSink(encoding, finalSink).get_ptr())); + } else { + // In this case we want to construct a TeeSink, to keep the error around + decompressionSink = std::make_shared{*decompressionSink}; + } } else { auto i = line.find(':'); if (i != string::npos) { @@ -352,8 +369,7 @@ struct curlFileTransfer : public FileTransfer if (writeException) failEx(writeException); - else if (code == CURLE_OK && - (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 0 /* other protocol */)) + else if (code == CURLE_OK && successfulStatuses.find(httpStatus) != successfulStatuses.end()) { result.cached = httpStatus == 304; act.progress(result.bodySize, result.bodySize); @@ -410,14 +426,14 @@ struct curlFileTransfer : public FileTransfer auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) + ? FileTransferError(Interrupted, nullptr, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) : httpStatus != 0 - ? FileTransferError(err, + ? FileTransferError(err, nullptr, fmt("unable to %s '%s': HTTP error %d", request.verb(), request.uri, httpStatus) + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) ) - : FileTransferError(err, + : FileTransferError(err, nullptr, fmt("unable to %s '%s': %s (%d)", request.verb(), request.uri, curl_easy_strerror(code), code)); @@ -675,7 +691,7 @@ struct curlFileTransfer : public FileTransfer auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; if (!s3Res.data) - throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); + throw FileTransferError(NotFound, nullptr, fmt("S3 object '%s' does not exist", request.uri)); res.data = s3Res.data; callback(std::move(res)); #else diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 11dca2fe0f0..44f550d6186 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -103,9 +103,10 @@ class FileTransferError : public Error { public: FileTransfer::Error error; + std::optional response; template - FileTransferError(FileTransfer::Error error, const Args & ... args) - : Error(args...), error(error) + FileTransferError(FileTransfer::Error error, std::optional response, const Args & ... args) + : Error(args...), error(error), response(response) { } }; diff --git a/src/libstore/references.cc b/src/libstore/references.cc index a10d536a355..4bb556c0a97 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -46,7 +46,7 @@ static void search(const unsigned char * s, size_t len, } -struct RefScanSink : Sink +struct RefScanSink : public Sink { HashSink hashSink; StringSet hashes; diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 6a848470520..92213adc969 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -295,7 +295,7 @@ void parseDump(ParseSink & sink, Source & source) } -struct RestoreSink : ParseSink +struct RestoreSink : public ParseSink { Path dstPath; AutoCloseFD fd; diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 768fe25368d..32d98a6104d 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -63,13 +63,6 @@ struct ParseSink virtual void createSymlink(const Path & path, const string & target) { }; }; -struct TeeSink : ParseSink -{ - TeeSource source; - - TeeSink(Source & source) : source(source) { } -}; - void parseDump(ParseSink & sink, Source & source); void restorePath(const Path & path, Source & source); diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index a117ddc72e1..86c8df8cb8d 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -18,7 +18,7 @@ namespace nix { // Don't feed brotli too much at once. -struct ChunkedCompressionSink : CompressionSink +struct ChunkedCompressionSink : public CompressionSink { uint8_t outbuf[32 * 1024]; @@ -36,7 +36,7 @@ struct ChunkedCompressionSink : CompressionSink virtual void writeInternal(const unsigned char * data, size_t len) = 0; }; -struct NoneSink : CompressionSink +struct NoneSink : public CompressionSink { Sink & nextSink; NoneSink(Sink & nextSink) : nextSink(nextSink) { } @@ -44,7 +44,7 @@ struct NoneSink : CompressionSink void write(const unsigned char * data, size_t len) override { nextSink(data, len); } }; -struct GzipDecompressionSink : CompressionSink +struct GzipDecompressionSink : public CompressionSink { Sink & nextSink; z_stream strm; @@ -104,7 +104,7 @@ struct GzipDecompressionSink : CompressionSink } }; -struct XzDecompressionSink : CompressionSink +struct XzDecompressionSink : public CompressionSink { Sink & nextSink; uint8_t outbuf[BUFSIZ]; @@ -156,7 +156,7 @@ struct XzDecompressionSink : CompressionSink } }; -struct BzipDecompressionSink : ChunkedCompressionSink +struct BzipDecompressionSink : public ChunkedCompressionSink { Sink & nextSink; bz_stream strm; @@ -209,7 +209,7 @@ struct BzipDecompressionSink : ChunkedCompressionSink } }; -struct BrotliDecompressionSink : ChunkedCompressionSink +struct BrotliDecompressionSink : public ChunkedCompressionSink { Sink & nextSink; BrotliDecoderState * state; @@ -285,7 +285,7 @@ ref makeDecompressionSink(const std::string & method, Sink & ne throw UnknownCompressionMethod("unknown compression method '%s'", method); } -struct XzCompressionSink : CompressionSink +struct XzCompressionSink : public CompressionSink { Sink & nextSink; uint8_t outbuf[BUFSIZ]; @@ -364,7 +364,7 @@ struct XzCompressionSink : CompressionSink } }; -struct BzipCompressionSink : ChunkedCompressionSink +struct BzipCompressionSink : public ChunkedCompressionSink { Sink & nextSink; bz_stream strm; @@ -417,7 +417,7 @@ struct BzipCompressionSink : ChunkedCompressionSink } }; -struct BrotliCompressionSink : ChunkedCompressionSink +struct BrotliCompressionSink : public ChunkedCompressionSink { Sink & nextSink; uint8_t outbuf[BUFSIZ]; diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index dd666a4e19f..9c68c1dbd30 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -8,7 +8,7 @@ namespace nix { -struct CompressionSink : BufferedSink +struct CompressionSink : public BufferedSink { virtual void finish() = 0; }; diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 180fb76334d..42f7335dbe2 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -125,7 +125,7 @@ string printHashType(HashType ht); union Ctx; -struct AbstractHashSink : virtual Sink +struct AbstractHashSink : public virtual Sink { virtual HashResult finish() = 0; }; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index a04118512fc..bfc18a8dba4 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -24,7 +24,7 @@ struct Sink /* A buffered abstract sink. */ -struct BufferedSink : virtual Sink +struct BufferedSink : public virtual Sink { size_t bufSize, bufPos; std::unique_ptr buffer; @@ -86,7 +86,7 @@ protected: /* A sink that writes data to a file descriptor. */ -struct FdSink : BufferedSink +struct FdSink : public BufferedSink { int fd; bool warn = false; @@ -144,7 +144,7 @@ private: /* A sink that writes data to a string. */ -struct StringSink : Sink +struct StringSink : public Sink { ref s; StringSink() : s(make_ref()) { }; @@ -181,6 +181,25 @@ struct TeeSource : Source } }; +struct TeeSink : public Sink +{ + Sink & orig; + ref data; + TeeSink(TeeSink &&) = default; + TeeSink(Sink & orig) + : orig(orig), data(make_ref()) { } + void operator () (const unsigned char * data, size_t len) { + this->data->append((const char *) data, len); + Sink::operator()(data, len); + } + void operator () (const std::string & s) + { + *data += s; + Sink::operator()(s); + } +}; + + /* A reader that consumes the original Source until 'size'. */ struct SizedSource : Source { @@ -213,7 +232,7 @@ struct SizedSource : Source }; /* Convert a function into a sink. */ -struct LambdaSink : Sink +struct LambdaSink : public Sink { typedef std::function lambda_t;