[RFC] allow configuring a compression algorithm for logs

doc/manual/src/release-notes/rl-next.md

@@ -0,0 +1,7 @@
+- The `compress-build-log` setting now accepts a compression algorithm name
+  (e.g. `zstd`, `gzip`, `xz`, `lz4`, `br`, `none`) instead of only `true`/`false`.
+  The default remains `bzip2` for backward compatibility.
+  `true` and `false` are still accepted as aliases for `bzip2` and `none`.
+
+  Reading build logs now supports all compression formats regardless of the
+  current setting, so logs remain readable after switching algorithms.

src/libstore/binary-cache-store.cc

@@ -170,13 +170,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     narInfo->fileHash = fileHash;
     narInfo->fileSize = fileSize;
     narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar"
-                   + (config.compression == CompressionAlgo::xz       ? ".xz"
-                      : config.compression == CompressionAlgo::bzip2  ? ".bz2"
-                      : config.compression == CompressionAlgo::zstd   ? ".zst"
-                      : config.compression == CompressionAlgo::lzip   ? ".lzip"
-                      : config.compression == CompressionAlgo::lz4    ? ".lz4"
-                      : config.compression == CompressionAlgo::brotli ? ".br"
-                                                                      : "");
+                   + std::string(compressionAlgoExtension(config.compression));
 
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
     printMsg(

src/libstore/build/derivation-building-goal.cc

@@ -1034,7 +1034,8 @@ LogFile::LogFile(Store & store, const StorePath & drvPath, const LogFileSettings
     Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, baseName.substr(0, 2));
     createDirs(dir);
 
-    Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2), logSettings.compressLog ? ".bz2" : "");
+    auto compression = logSettings.compressLog.get();
+    Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2), compressionAlgoExtension(compression));
 
     fd = openNewFileForWrite(
         logFileName,
@@ -1048,8 +1049,8 @@ LogFile::LogFile(Store & store, const StorePath & drvPath, const LogFileSettings
 
     fileSink = std::make_shared<FdSink>(fd.get());
 
-    if (logSettings.compressLog)
-        sink = std::shared_ptr<CompressionSink>(makeCompressionSink(CompressionAlgo::bzip2, *fileSink));
+    if (logSettings.compressLog.enabled())
+        sink = std::shared_ptr<CompressionSink>(makeCompressionSink(compression, *fileSink));
     else
         sink = fileSink;
 }

src/libstore/globals.cc

@@ -74,6 +74,37 @@ LogFileSettings::LogFileSettings()
 {
 }
 
+CompressionAlgo BuildLogCompressionSetting::parse(const std::string & str) const
+{
+    if (str == "true")
+        return CompressionAlgo::bzip2;
+    if (str == "false")
+        return CompressionAlgo::none;
+    try {
+        return parseCompressionAlgo(str, /*suggestions=*/true);
+    } catch (UnknownCompressionMethod & e) {
+        throw UsageError(e.info().suggestions, "option '%s' has invalid value '%s'", name, str);
+    }
+}
+
+void BuildLogCompressionSetting::convertToArg(Args & args, const std::string & category)
+{
+    args.addFlag({
+        .longName = name,
+        .aliases = aliases,
+        .description = fmt("Compression method for build logs."),
+        .category = category,
+        .labels = {"method"},
+        .handler = {[this](std::string s) { override(parse(s)); }},
+    });
+    args.addFlag({
+        .longName = "no-" + name,
+        .description = fmt("Disable build log compression."),
+        .category = category,
+        .handler = {[this]() { override(CompressionAlgo::none); }},
+    });
+}
+
 Settings settings;
 
 static GlobalConfig::Register rSettings(&settings);

src/libstore/include/nix/store/globals.hh

@@ -9,10 +9,41 @@
 #include "nix/store/build/derivation-builder.hh"
 #include "nix/store/local-settings.hh"
 
+#include "nix/util/compression-settings.hh"
 #include "nix/store/config.hh"
 
 namespace nix {
 
+/**
+ * Custom setting subclass for build log compression that handles
+ * backward compatibility with the old boolean values.
+ *
+ * Accepts `true` (mapped to `bzip2`), `false` (mapped to `none`),
+ * or any compression algorithm name.
+ */
+struct BuildLogCompressionSetting : public BaseSetting<CompressionAlgo>
+{
+    BuildLogCompressionSetting(
+        Config * options,
+        CompressionAlgo def,
+        const std::string & name,
+        const std::string & description,
+        const StringSet & aliases = {})
+        : BaseSetting<CompressionAlgo>(def, true, name, description, aliases)
+    {
+        options->addSetting(this);
+    }
+
+    CompressionAlgo parse(const std::string & str) const override;
+
+    void convertToArg(Args & args, const std::string & category) override;
+
+    bool enabled() const
+    {
+        return value != CompressionAlgo::none;
+    }
+};
+
 struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
 {
     MaxBuildJobsSetting(
@@ -52,14 +83,16 @@ public:
         )",
         {"build-keep-log"}};
 
-    Setting<bool> compressLog{
+    BuildLogCompressionSetting compressLog{
         this,
-        true,
+        CompressionAlgo::bzip2,
         "compress-build-log",
         R"(
-          If set to `true` (the default), build logs written to
-          `/nix/var/log/nix/drvs` are compressed on the fly using bzip2.
-          Otherwise, they are not compressed.
+          Compression method for build logs written to `/nix/var/log/nix/drvs`.
+          Valid values are `none` (no compression), `bzip2` (the default),
+          `zstd`, `xz`, `gzip`, `lz4`, or `br`.
+          For backward compatibility, `true` is equivalent to `bzip2`
+          and `false` is equivalent to `none`.
         )",
         {"build-compress-log"}};
 };

src/libstore/local-fs-store.cc

@@ -118,20 +118,31 @@ std::optional<std::string> LocalFSStore::getBuildLogExact(const StorePath & path
 {
     auto baseName = path.to_string();
 
+    static const std::vector<std::pair<std::string, CompressionAlgo>> compressedExtensions = {
+        {".bz2", CompressionAlgo::bzip2},
+        {".zst", CompressionAlgo::zstd},
+        {".xz", CompressionAlgo::xz},
+        {".gz", CompressionAlgo::gzip},
+        {".lz4", CompressionAlgo::lz4},
+        {".br", CompressionAlgo::brotli},
+    };
+
     for (int j = 0; j < 2; j++) {
 
         Path logPath =
             j == 0 ? fmt("%s/%s/%s/%s", config.logDir.get(), drvsLogDir, baseName.substr(0, 2), baseName.substr(2))
                    : fmt("%s/%s/%s", config.logDir.get(), drvsLogDir, baseName);
-        Path logBz2Path = logPath + ".bz2";
 
         if (pathExists(logPath))
             return readFile(logPath);
 
-        else if (pathExists(logBz2Path)) {
-            try {
-                return decompress("bzip2", readFile(logBz2Path));
-            } catch (Error &) {
+        for (auto & [ext, algo] : compressedExtensions) {
+            Path compressedPath = logPath + ext;
+            if (pathExists(compressedPath)) {
+                try {
+                    return decompress(showCompressionAlgo(algo), readFile(compressedPath));
+                } catch (Error &) {
+                }
             }
         }
     }

src/libstore/local-store.cc

@@ -1578,7 +1578,16 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
 
     auto baseName = drvPath.to_string();
 
-    auto logPath = fmt("%s/%s/%s/%s.bz2", config->logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2));
+    auto & logSettings = settings.getLogFileSettings();
+    auto compression = logSettings.compressLog.get();
+
+    auto logPath =
+        fmt("%s/%s/%s/%s%s",
+            config->logDir,
+            drvsLogDir,
+            baseName.substr(0, 2),
+            baseName.substr(2),
+            compressionAlgoExtension(compression));
 
     if (pathExists(logPath))
         return;
@@ -1587,7 +1596,10 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
 
     auto tmpFile = fmt("%s.tmp.%d", logPath, getpid());
 
-    writeFile(tmpFile, compress(CompressionAlgo::bzip2, log));
+    if (logSettings.compressLog.enabled())
+        writeFile(tmpFile, compress(compression, log));
+    else
+        writeFile(tmpFile, log);
 
     std::filesystem::rename(tmpFile, logPath);
 }

src/libutil/compression-algo.cc

@@ -43,4 +43,37 @@ std::string showCompressionAlgo(CompressionAlgo method)
     unreachable();
 }
 
+std::string_view compressionAlgoExtension(CompressionAlgo method)
+{
+    switch (method) {
+    case CompressionAlgo::none:
+        return "";
+    case CompressionAlgo::brotli:
+        return ".br";
+    case CompressionAlgo::bzip2:
+        return ".bz2";
+    case CompressionAlgo::compress:
+        return ".Z";
+    case CompressionAlgo::grzip:
+        return ".grz";
+    case CompressionAlgo::gzip:
+        return ".gz";
+    case CompressionAlgo::lrzip:
+        return ".lrz";
+    case CompressionAlgo::lz4:
+        return ".lz4";
+    case CompressionAlgo::lzip:
+        return ".lzip";
+    case CompressionAlgo::lzma:
+        return ".lzma";
+    case CompressionAlgo::lzop:
+        return ".lzo";
+    case CompressionAlgo::xz:
+        return ".xz";
+    case CompressionAlgo::zstd:
+        return ".zst";
+    }
+    unreachable();
+}
+
 } // namespace nix

src/libutil/include/nix/util/compression-algo.hh

@@ -37,6 +37,12 @@ CompressionAlgo parseCompressionAlgo(std::string_view method, bool suggestions =
 
 std::string showCompressionAlgo(CompressionAlgo method);
 
+/**
+ * Returns the file extension (including the dot) for the given
+ * compression algorithm. Returns empty string for `none`.
+ */
+std::string_view compressionAlgoExtension(CompressionAlgo method);
+
 MakeError(UnknownCompressionMethod, Error);
 
 } // namespace nix

tests/functional/logging.sh

@@ -11,11 +11,25 @@ path=$(nix-build dependencies.nix --no-out-link)
 # Test nix-store -l.
 [ "$(nix-store -l "$path")" = FOO ]
 
-# Test compressed logs.
+# Test compressed logs (bzip2).
 clearStore
 rm -rf "$NIX_LOG_DIR"
 (! nix-store -l "$path")
-nix-build dependencies.nix --no-out-link --compress-build-log
+nix-build dependencies.nix --no-out-link --compress-build-log bzip2
+[ "$(nix-store -l "$path")" = FOO ]
+
+# Test compressed logs (zstd).
+clearStore
+rm -rf "$NIX_LOG_DIR"
+(! nix-store -l "$path")
+nix-build dependencies.nix --no-out-link --compress-build-log zstd
+[ "$(nix-store -l "$path")" = FOO ]
+
+# Test no compression.
+clearStore
+rm -rf "$NIX_LOG_DIR"
+(! nix-store -l "$path")
+nix-build dependencies.nix --no-out-link --no-compress-build-log
 [ "$(nix-store -l "$path")" = FOO ]
 
 # test whether empty logs work fine with `nix log`.