From 4d144844019b3a601baa75387c3a50cb9e659a16 Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Thu, 5 Jul 2018 01:08:44 -0500 Subject: [PATCH] add support for specifying 'compression-level' for NAR's --- src/libstore/binary-cache-store.cc | 2 +- src/libstore/binary-cache-store.hh | 6 ++- src/libutil/compression.cc | 75 ++++++++++++++++++++---------- src/libutil/compression.hh | 4 +- 4 files changed, 59 insertions(+), 28 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 76c0a1a891b..9e34361f720 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -170,7 +170,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const refcompression = compression; auto now1 = std::chrono::steady_clock::now(); - auto narCompressed = compress(compression, *nar, parallelCompression); + auto narCompressed = compress(compression, *nar, parallelCompression, compressionLevel); auto now2 = std::chrono::steady_clock::now(); narInfo->fileHash = hashString(htSHA256, *narCompressed); narInfo->fileSize = narCompressed->size(); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 6bc83fc50ca..34f1e62796b 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -20,7 +20,11 @@ public: const Setting secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; const Setting localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; const Setting parallelCompression{this, false, "parallel-compression", - "enable multi-threading compression, available for xz only currently"}; + "enable multi-threading compression for NARs, available for xz only currently"}; + const Setting compressionLevel{this, -1, "compression-level", + "specify 'preset level' of compression to be used with NARs: " + "meaning and accepted range of values depends on compression method selected, " + "other than -1 which we reserve to indicate Nix defaults should be used"}; private: diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index e1782f8c4bd..e904a702302 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -19,6 +19,22 @@ namespace nix { static const size_t bufSize = 32 * 1024; +static const int COMPRESSION_LEVEL_DEFAULT = -1; + +static unsigned checkLevel(unsigned min, unsigned max, unsigned methodDefault, std::string method, int level) { + if (level == COMPRESSION_LEVEL_DEFAULT) + return methodDefault; + if (level < 0) + throw CompressionError("compression level must be a non-negative integer"); + + unsigned l = static_cast(level); + if (min <= l && l <= max) + return l; + + throw CompressionError("requested compression level '%u' not valid for method '%s': must be [%u,%u] (default=%u)", + l, method, min, max, methodDefault); +} + static void decompressNone(Source & source, Sink & sink) { std::vector buf(bufSize); @@ -221,7 +237,10 @@ void decompress(const std::string & method, Source & source, Sink & sink) struct NoneSink : CompressionSink { Sink & nextSink; - NoneSink(Sink & nextSink) : nextSink(nextSink) { } + NoneSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink) { + if (level != COMPRESSION_LEVEL_DEFAULT) + printError("Warning: requested compression level '%d' not supported by compression method 'none'", level); + } void finish() override { flush(); } void write(const unsigned char * data, size_t len) override { nextSink(data, len); } }; @@ -234,8 +253,8 @@ struct XzSink : CompressionSink bool finished = false; template - XzSink(Sink & nextSink, F&& initEncoder) : nextSink(nextSink) { - lzma_ret ret = initEncoder(); + XzSink(Sink & nextSink, F&& initEncoder, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink) { + lzma_ret ret = initEncoder(checkLevel(0, 9, LZMA_PRESET_DEFAULT, "xz", level)); if (ret != LZMA_OK) throw CompressionError("unable to initialise lzma encoder"); // FIXME: apply the x86 BCJ filter? @@ -243,9 +262,9 @@ struct XzSink : CompressionSink strm.next_out = outbuf; strm.avail_out = sizeof(outbuf); } - XzSink(Sink & nextSink) : XzSink(nextSink, [this]() { - return lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64); - }) {} + XzSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT) : XzSink(nextSink, [this](unsigned level) { + return lzma_easy_encoder(&strm, level, LZMA_CHECK_CRC64); + }, level) {} ~XzSink() { @@ -302,11 +321,11 @@ struct XzSink : CompressionSink #ifdef HAVE_LZMA_MT struct ParallelXzSink : public XzSink { - ParallelXzSink(Sink &nextSink) : XzSink(nextSink, [this]() { + ParallelXzSink(Sink &nextSink, int level) : XzSink(nextSink, [this](unsigned level) { lzma_mt mt_options = {}; mt_options.flags = 0; mt_options.timeout = 300; // Using the same setting as the xz cmd line - mt_options.preset = LZMA_PRESET_DEFAULT; + mt_options.preset = level; mt_options.filters = NULL; mt_options.check = LZMA_CHECK_CRC64; mt_options.threads = lzma_cputhreads(); @@ -316,7 +335,7 @@ struct ParallelXzSink : public XzSink // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the // number of threads. return lzma_stream_encoder_mt(&strm, &mt_options); - }) {} + }, level) {} }; #endif @@ -327,10 +346,11 @@ struct BzipSink : CompressionSink bz_stream strm; bool finished = false; - BzipSink(Sink & nextSink) : nextSink(nextSink) + BzipSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink) { memset(&strm, 0, sizeof(strm)); - int ret = BZ2_bzCompressInit(&strm, 9, 0, 30); + auto l = checkLevel(1, 9, 9, "bzip2", level); + int ret = BZ2_bzCompressInit(&strm, l, 0, 30); if (ret != BZ_OK) throw CompressionError("unable to initialise bzip2 encoder"); @@ -430,9 +450,11 @@ struct LambdaCompressionSink : CompressionSink struct BrotliCmdSink : LambdaCompressionSink { - BrotliCmdSink(Sink& nextSink) - : LambdaCompressionSink(nextSink, [](const std::string& data) { - return runProgram(BROTLI, true, {}, data); + BrotliCmdSink(Sink& nextSink, int level = COMPRESSION_LEVEL_DEFAULT) + : LambdaCompressionSink(nextSink, [level](const std::string& data) { + // Hard-code values from brotli manpage + std::string levelArg = fmt("-%u", checkLevel(0, 11, 11, "brotli", level)); + return runProgram(BROTLI, true, {levelArg}, data); }) { } @@ -446,11 +468,16 @@ struct BrotliSink : CompressionSink BrotliEncoderState *state; bool finished = false; - BrotliSink(Sink & nextSink) : nextSink(nextSink) + BrotliSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink) { state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); if (!state) throw CompressionError("unable to initialise brotli encoder"); + + if (!BrotliEncoderSetParameter(state, BROTLI_PARAM_QUALITY, + checkLevel(BROTLI_MIN_QUALITY, BROTLI_MAX_QUALITY, + BROTLI_DEFAULT_QUALITY, "brotli", level))) + throw CompressionError("failure setting requested compression level for brotli encoder"); } ~BrotliSink() @@ -527,36 +554,36 @@ struct BrotliSink : CompressionSink }; #endif // HAVE_BROTLI -ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel) +ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level) { if (parallel) { #ifdef HAVE_LZMA_MT if (method == "xz") - return make_ref(nextSink); + return make_ref(nextSink, level); #endif printMsg(lvlError, format("Warning: parallel compression requested but not supported for method '%1%', falling back to single-threaded compression") % method); } if (method == "none") - return make_ref(nextSink); + return make_ref(nextSink, level); else if (method == "xz") - return make_ref(nextSink); + return make_ref(nextSink, level); else if (method == "bzip2") - return make_ref(nextSink); + return make_ref(nextSink, level); else if (method == "br") #if HAVE_BROTLI - return make_ref(nextSink); + return make_ref(nextSink, level); #else - return make_ref(nextSink); + return make_ref(nextSink, level); #endif else throw UnknownCompressionMethod(format("unknown compression method '%s'") % method); } -ref compress(const std::string & method, const std::string & in, const bool parallel) +ref compress(const std::string & method, const std::string & in, const bool parallel, int level) { StringSink ssink; - auto sink = makeCompressionSink(method, ssink, parallel); + auto sink = makeCompressionSink(method, ssink, parallel, level); (*sink)(in); sink->finish(); return ssink.s; diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index f7a3e3fbd32..6f5a086049f 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -12,14 +12,14 @@ ref decompress(const std::string & method, const std::string & in); void decompress(const std::string & method, Source & source, Sink & sink); -ref compress(const std::string & method, const std::string & in, const bool parallel = false); +ref compress(const std::string & method, const std::string & in, const bool parallel = false, int level = -1); struct CompressionSink : BufferedSink { virtual void finish() = 0; }; -ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false); +ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1); MakeError(UnknownCompressionMethod, Error);