add support for specifying 'compression-level' for NAR's

NixOS · Jul 5, 2018 · 4d14484 · 4d14484
1 parent f1f4c25
commit 4d14484
Show file tree

Hide file tree

Showing 4 changed files with 59 additions and 28 deletions.
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
@@ -170,7 +170,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     /* Compress the NAR. */
     narInfo->compression = compression;
     auto now1 = std::chrono::steady_clock::now();
-    auto narCompressed = compress(compression, *nar, parallelCompression);
+    auto narCompressed = compress(compression, *nar, parallelCompression, compressionLevel);
     auto now2 = std::chrono::steady_clock::now();
     narInfo->fileHash = hashString(htSHA256, *narCompressed);
     narInfo->fileSize = narCompressed->size();

diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
@@ -20,7 +20,11 @@ public:
     const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
     const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
     const Setting<bool> parallelCompression{this, false, "parallel-compression",
-        "enable multi-threading compression, available for xz only currently"};
+        "enable multi-threading compression for NARs, available for xz only currently"};
+    const Setting<int> compressionLevel{this, -1, "compression-level",
+        "specify 'preset level' of compression to be used with NARs: "
+        "meaning and accepted range of values depends on compression method selected, "
+        "other than -1 which we reserve to indicate Nix defaults should be used"};
 
 private:
 

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
@@ -19,6 +19,22 @@ namespace nix {
 
 static const size_t bufSize = 32 * 1024;
 
+static const int COMPRESSION_LEVEL_DEFAULT = -1;
+
+static unsigned checkLevel(unsigned min, unsigned max, unsigned methodDefault, std::string method, int level) {
+    if (level == COMPRESSION_LEVEL_DEFAULT)
+        return methodDefault;
+    if (level < 0)
+        throw CompressionError("compression level must be a non-negative integer");
+
+    unsigned l = static_cast<unsigned>(level);
+    if (min <= l && l <= max)
+        return l;
+
+    throw CompressionError("requested compression level '%u' not valid for method '%s': must be [%u,%u] (default=%u)",
+            l, method, min, max, methodDefault);
+}
+
 static void decompressNone(Source & source, Sink & sink)
 {
     std::vector<unsigned char> buf(bufSize);
@@ -221,7 +237,10 @@ void decompress(const std::string & method, Source & source, Sink & sink)
 struct NoneSink : CompressionSink
 {
     Sink & nextSink;
-    NoneSink(Sink & nextSink) : nextSink(nextSink) { }
+    NoneSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink) {
+        if (level != COMPRESSION_LEVEL_DEFAULT)
+            printError("Warning: requested compression level '%d' not supported by compression method 'none'", level);
+    }
     void finish() override { flush(); }
     void write(const unsigned char * data, size_t len) override { nextSink(data, len); }
 };
@@ -234,18 +253,18 @@ struct XzSink : CompressionSink
     bool finished = false;
 
     template <typename F>
-    XzSink(Sink & nextSink, F&& initEncoder) : nextSink(nextSink) {
-        lzma_ret ret = initEncoder();
+    XzSink(Sink & nextSink, F&& initEncoder, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink) {
+        lzma_ret ret = initEncoder(checkLevel(0, 9, LZMA_PRESET_DEFAULT, "xz", level));
         if (ret != LZMA_OK)
             throw CompressionError("unable to initialise lzma encoder");
         // FIXME: apply the x86 BCJ filter?
 
         strm.next_out = outbuf;
         strm.avail_out = sizeof(outbuf);
     }
-    XzSink(Sink & nextSink) : XzSink(nextSink, [this]() {
-        return lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64);
-    }) {}
+    XzSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT) : XzSink(nextSink, [this](unsigned level) {
+        return lzma_easy_encoder(&strm, level, LZMA_CHECK_CRC64);
+    }, level) {}
 
     ~XzSink()
     {
@@ -302,11 +321,11 @@ struct XzSink : CompressionSink
 #ifdef HAVE_LZMA_MT
 struct ParallelXzSink : public XzSink
 {
-  ParallelXzSink(Sink &nextSink) : XzSink(nextSink, [this]() {
+  ParallelXzSink(Sink &nextSink, int level) : XzSink(nextSink, [this](unsigned level) {
         lzma_mt mt_options = {};
         mt_options.flags = 0;
         mt_options.timeout = 300; // Using the same setting as the xz cmd line
-        mt_options.preset = LZMA_PRESET_DEFAULT;
+        mt_options.preset = level;
         mt_options.filters = NULL;
         mt_options.check = LZMA_CHECK_CRC64;
         mt_options.threads = lzma_cputhreads();
@@ -316,7 +335,7 @@ struct ParallelXzSink : public XzSink
         // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
         // number of threads.
         return lzma_stream_encoder_mt(&strm, &mt_options);
-  }) {}
+  }, level) {}
 };
 #endif
 
@@ -327,10 +346,11 @@ struct BzipSink : CompressionSink
     bz_stream strm;
     bool finished = false;
 
-    BzipSink(Sink & nextSink) : nextSink(nextSink)
+    BzipSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink)
     {
         memset(&strm, 0, sizeof(strm));
-        int ret = BZ2_bzCompressInit(&strm, 9, 0, 30);
+        auto l = checkLevel(1, 9, 9, "bzip2", level);
+        int ret = BZ2_bzCompressInit(&strm, l, 0, 30);
         if (ret != BZ_OK)
             throw CompressionError("unable to initialise bzip2 encoder");
 
@@ -430,9 +450,11 @@ struct LambdaCompressionSink : CompressionSink
 
 struct BrotliCmdSink : LambdaCompressionSink
 {
-    BrotliCmdSink(Sink& nextSink)
-        : LambdaCompressionSink(nextSink, [](const std::string& data) {
-            return runProgram(BROTLI, true, {}, data);
+    BrotliCmdSink(Sink& nextSink, int level = COMPRESSION_LEVEL_DEFAULT)
+        : LambdaCompressionSink(nextSink, [level](const std::string& data) {
+            // Hard-code values from brotli manpage
+            std::string levelArg = fmt("-%u", checkLevel(0, 11, 11, "brotli", level));
+            return runProgram(BROTLI, true, {levelArg}, data);
         })
     {
     }
@@ -446,11 +468,16 @@ struct BrotliSink : CompressionSink
     BrotliEncoderState *state;
     bool finished = false;
 
-    BrotliSink(Sink & nextSink) : nextSink(nextSink)
+    BrotliSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink)
     {
         state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr);
         if (!state)
             throw CompressionError("unable to initialise brotli encoder");
+
+        if (!BrotliEncoderSetParameter(state, BROTLI_PARAM_QUALITY,
+                checkLevel(BROTLI_MIN_QUALITY, BROTLI_MAX_QUALITY,
+                    BROTLI_DEFAULT_QUALITY, "brotli", level)))
+            throw CompressionError("failure setting requested compression level for brotli encoder");
     }
 
     ~BrotliSink()
@@ -527,36 +554,36 @@ struct BrotliSink : CompressionSink
 };
 #endif // HAVE_BROTLI
 
-ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
+ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level)
 {
     if (parallel) {
 #ifdef HAVE_LZMA_MT
         if (method == "xz")
-            return make_ref<ParallelXzSink>(nextSink);
+            return make_ref<ParallelXzSink>(nextSink, level);
 #endif
         printMsg(lvlError, format("Warning: parallel compression requested but not supported for method '%1%', falling back to single-threaded compression") % method);
     }
 
     if (method == "none")
-        return make_ref<NoneSink>(nextSink);
+        return make_ref<NoneSink>(nextSink, level);
     else if (method == "xz")
-        return make_ref<XzSink>(nextSink);
+        return make_ref<XzSink>(nextSink, level);
     else if (method == "bzip2")
-        return make_ref<BzipSink>(nextSink);
+        return make_ref<BzipSink>(nextSink, level);
     else if (method == "br")
 #if HAVE_BROTLI
-        return make_ref<BrotliSink>(nextSink);
+        return make_ref<BrotliSink>(nextSink, level);
 #else
-        return make_ref<BrotliCmdSink>(nextSink);
+        return make_ref<BrotliCmdSink>(nextSink, level);
 #endif
     else
         throw UnknownCompressionMethod(format("unknown compression method '%s'") % method);
 }
 
-ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel)
+ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel, int level)
 {
     StringSink ssink;
-    auto sink = makeCompressionSink(method, ssink, parallel);
+    auto sink = makeCompressionSink(method, ssink, parallel, level);
     (*sink)(in);
     sink->finish();
     return ssink.s;

diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh
@@ -12,14 +12,14 @@ ref<std::string> decompress(const std::string & method, const std::string & in);
 
 void decompress(const std::string & method, Source & source, Sink & sink);
 
-ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
+ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false, int level = -1);
 
 struct CompressionSink : BufferedSink
 {
     virtual void finish() = 0;
 };
 
-ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false);
+ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1);
 
 MakeError(UnknownCompressionMethod, Error);