Enable lz4 compression levels 3..12

src/lib/Makefile.am

@@ -10,7 +10,7 @@ LDADD = $(DEPS_LIBS)
 # libnfdump sources
 util = util.c util.h
 pidfile = pidfile.c pidfile.h
-filelzo = minilzo.c minilzo.h lzoconf.h lzodefs.h lz4.c lz4.h
+compress = minilzo.c minilzo.h lzoconf.h lzodefs.h lz4.c lz4.h lz4hc.c lz4hc.h
 nffile = nffile.c nffile.h nffileV2.h queue.c queue.h nfx.c nfx.h nfxV3.h nfxV3.c id.h
 
 if NEEDFTSCOMPAT
@@ -28,7 +28,7 @@ vcs_track.h: Makefile
 	./gen_version.sh
 
 lib_LTLIBRARIES = libnfdump.la
-libnfdump_la_SOURCES = $(util) $(pidfile) $(filelzo) $(nffile) $(nflist) $(filter) $(output) $(regex) $(daemon) $(version) vcs_track.h
+libnfdump_la_SOURCES = $(util) $(pidfile) $(compress) $(nffile) $(nflist) $(filter) $(output) $(regex) $(daemon) $(version) vcs_track.h
 libnfdump_la_LDFLAGS = -release @VERSION@
 
 EXTRA_DIST = gen_version.sh

src/lib/nffile.c

@@ -55,6 +55,7 @@
 #include "config.h"
 #include "flist.h"
 #include "lz4.h"
+#include "lz4hc.h"
 #include "minilzo.h"
 #include "nfdump.h"
 #include "nffileV2.h"
@@ -84,7 +85,7 @@ static int Compress_Block_LZO(dataBlock_t *in_block, dataBlock_t *out_block, siz
 
 static int Uncompress_Block_LZO(dataBlock_t *in_block, dataBlock_t *out_block, size_t block_size);
 
-static int Compress_Block_LZ4(dataBlock_t *in_block, dataBlock_t *out_block, size_t block_size);
+static int Compress_Block_LZ4(dataBlock_t *in_block, dataBlock_t *out_block, size_t block_size, int level);
 
 static int Uncompress_Block_LZ4(dataBlock_t *in_block, dataBlock_t *out_block, size_t block_size);
 
@@ -142,7 +143,7 @@ int Init_nffile(queue_t *fileList) {
 
 }  // End of Init_nffile
 
-int ParseCompression(char *arg, uint32_t *level) {
+int ParseCompression(char *arg) {
     if (arg == NULL) {
         return LZO_COMPRESSED;
     }
@@ -153,23 +154,21 @@ int ParseCompression(char *arg, uint32_t *level) {
         return -1;
     }
 
-    *level = 0;
+    int level = 0;
     char *s = strchr(arg, ':');
     if (s) {
         *s++ = '\0';
-        uint32_t l = 0;
         while (*s && isdigit(*s)) {
-            l = 10 * l + (*s++ - 0x30);
+            level = 10 * level + (*s++ - 0x30);
         }
         if (*s) {
             LogError("Invalid compression level: %s", s);
             return -1;
         }
-        if (l > 100) {
-            LogError("Invalid compression level: %u", l);
+        if (level > 100) {
+            LogError("Invalid compression level: %u", level);
             return -1;
         }
-        *level = l;
     }
 
     for (int i = 0; arg[i]; i++) {
@@ -178,7 +177,14 @@ int ParseCompression(char *arg, uint32_t *level) {
 
     if (strcmp(arg, "0") == 0) return NOT_COMPRESSED;
     if (strcmp(arg, "lzo") == 0 || strcmp(arg, "1") == 0) return LZO_COMPRESSED;
-    if (strcmp(arg, "lz4") == 0 || strcmp(arg, "3") == 0) return LZ4_COMPRESSED;
+    if (strcmp(arg, "lz4") == 0 || strcmp(arg, "3") == 0) {
+        if (level <= LZ4HC_CLEVEL_MAX) {
+            return (level << 16) | LZ4_COMPRESSED;
+        } else {
+            LogError("LZ4 max compression level is %d", LZ4HC_CLEVEL_MAX);
+            return -1;
+        }
+    }
     if (strcmp(arg, "bz2") == 0 || strcmp(arg, "bzip2") == 0 || strcmp(arg, "2") == 0) return BZ2_COMPRESSED;
 
     // anything else is invalid
@@ -278,12 +284,17 @@ static int Uncompress_Block_LZO(dataBlock_t *in_block, dataBlock_t *out_block, s
 
 }  // End of Uncompress_Block_LZO
 
-static int Compress_Block_LZ4(dataBlock_t *in_block, dataBlock_t *out_block, size_t block_size) {
+static int Compress_Block_LZ4(dataBlock_t *in_block, dataBlock_t *out_block, size_t block_size, int level) {
     const char *in = (const char *)((void *)in_block + sizeof(dataBlock_t));
     char *out = (char *)((void *)out_block + sizeof(dataBlock_t));
     int in_len = in_block->size;
 
-    int out_len = LZ4_compress_default(in, out, in_len, block_size);
+    int out_len;
+    if (level > LZ4HC_CLEVEL_MIN)
+        out_len = LZ4_compress_HC(in, out, in_len, block_size, level);
+    else
+        out_len = LZ4_compress_default(in, out, in_len, block_size);
+
     if (out_len == 0) {
         LogError("Compress_Block_LZ4() error compression aborted in %s line %d: LZ4 : buffer too small", __FILE__, __LINE__);
         return -1;
@@ -1248,7 +1259,7 @@ static int nfwrite(nffile_t *nffile, dataBlock_t *block_header) {
             break;
         case LZ4_COMPRESSED:
             buff = NewDataBlock();
-            if (Compress_Block_LZ4(block_header, buff, nffile->buff_size) < 0) failed = 1;
+            if (Compress_Block_LZ4(block_header, buff, nffile->buff_size, level) < 0) failed = 1;
             wptr = buff;
             break;
         case BZ2_COMPRESSED:

src/lib/nffile.h

@@ -244,7 +244,7 @@ typedef struct record_header_s {
 
 int Init_nffile(queue_t *fileList);
 
-int ParseCompression(char *arg, uint32_t *level);
+int ParseCompression(char *arg);
 
 unsigned ReportBlocks(void);
 

src/nfcapd/nfcapd.c

@@ -597,7 +597,6 @@ int main(int argc, char **argv) {
     time_t twin;
     int sock, do_daemonize, expire, spec_time_extension;
     int subdir_index, sampling_rate, compress, srcSpoofing;
-    uint32_t compression_level;
 #ifdef PCAP
     char *pcap_file = NULL;
     char *pcap_device = NULL;
@@ -621,7 +620,6 @@ int main(int argc, char **argv) {
     expire = 0;
     sampling_rate = 1;
     compress = NOT_COMPRESSED;
-    compression_level = 0;
     memset((void *)&repeater, 0, sizeof(repeater));
     srcSpoofing = 0;
     configFile = NULL;
@@ -843,7 +841,7 @@ int main(int argc, char **argv) {
                 if (optarg == NULL) {
                     compress = LZO_COMPRESSED;
                 } else {
-                    compress = ParseCompression(optarg, &compression_level);
+                    compress = ParseCompression(optarg);
                 }
                 if (compress == -1) {
                     LogError("Usage for option -z: set -z=lzo, -z=lz4 or -z=bz2 for valid compression formats");

src/nfdump/nfdump.c

@@ -570,7 +570,7 @@ int main(int argc, char **argv) {
     int flow_stat, aggregate, aggregate_mask, bidir;
     int print_stat, gnuplot_stat, syntax_only, compress;
     int GuessDir, ModifyCompress;
-    uint32_t limitRecords, compression_level;
+    uint32_t limitRecords;
     char Ident[IDENTLEN];
     flist_t flist;
 
@@ -588,7 +588,6 @@ int main(int argc, char **argv) {
     limitRecords = 0;
     skipped_blocks = 0;
     compress = NOT_COMPRESSED;
-    compression_level = 0;
     GuessDir = 0;
     nameserver = NULL;
 
@@ -709,7 +708,7 @@ int main(int argc, char **argv) {
                 if (optarg == NULL) {
                     compress = LZO_COMPRESSED;
                 } else {
-                    compress = ParseCompression(optarg, &compression_level);
+                    compress = ParseCompression(optarg);
                 }
                 if (compress == -1) {
                     LogError("Usage for option -z: set -z=lzo, -z=lz4 or -z=bz2 for valid compression formats");
@@ -820,7 +819,7 @@ int main(int argc, char **argv) {
                 }
                 break;
             case 'J':
-                ModifyCompress = ParseCompression(optarg, &compression_level);
+                ModifyCompress = ParseCompression(optarg);
                 if (ModifyCompress < 0) {
                     LogError("Expected -J <arg>, 0 for uncompressed, 1, LZO, 2, BZ2, 3, LZ4");
                     exit(EXIT_FAILURE);

src/nfpcapd/nfpcapd.c

@@ -288,7 +288,6 @@ int main(int argc, char *argv[]) {
     char *device, *pcapfile, *filter, *datadir, *pcap_datadir, *pidfile, *configFile, *options;
     char *Ident, *userid, *groupid, *metricsocket;
     char *time_extension;
-    uint32_t compression_level;
 
     snaplen = 1522;
     bufflen = 0;
@@ -311,7 +310,6 @@ int main(int argc, char *argv[]) {
     time_extension = "%Y%m%d%H%M";
     subdir_index = 0;
     compress = NOT_COMPRESSED;
-    compression_level = 0;
     verbose = 0;
     expire = 0;
     cache_size = 0;
@@ -495,7 +493,7 @@ int main(int argc, char *argv[]) {
                 if (optarg == NULL) {
                     compress = LZO_COMPRESSED;
                 } else {
-                    compress = ParseCompression(optarg, &compression_level);
+                    compress = ParseCompression(optarg);
                 }
                 if (compress == -1) {
                     LogError("Usage for option -z: set -z=lzo, -z=lz4 or -z=bz2 for valid compression formats");

src/nfsen/nfprofile.c

@@ -432,7 +432,6 @@ static profile_param_info_t *ParseParams(char *profile_datadir) {
 
 int main(int argc, char **argv) {
     unsigned int num_channels, compress;
-    uint32_t compression_level;
     profile_param_info_t *profile_list;
     char *ffile, *filename, *syslog_facility;
     char *profile_datadir, *profile_statdir, *nameserver;
@@ -446,7 +445,6 @@ int main(int argc, char **argv) {
     tslot = 0;
     syntax_only = 0;
     compress = NOT_COMPRESSED;
-    compression_level = 0;
     subdir_index = 0;
     profile_list = NULL;
     nameserver = NULL;
@@ -532,7 +530,7 @@ int main(int argc, char **argv) {
                 if (optarg == NULL) {
                     compress = LZO_COMPRESSED;
                 } else {
-                    compress = ParseCompression(optarg, &compression_level);
+                    compress = ParseCompression(optarg);
                 }
                 if (compress == -1) {
                     LogError("Usage for option -z: set -z=lzo, -z=lz4 or -z=bz2 for valid compression formats");

src/sflow/sfcapd.c

@@ -565,7 +565,6 @@ int main(int argc, char **argv) {
     time_t twin;
     int sock, do_daemonize, expire, spec_time_extension;
     int subdir_index, compress, srcSpoofing;
-    uint32_t compression_level;
 #ifdef PCAP
     char *pcap_file = NULL;
     char *pcap_device = NULL;
@@ -588,7 +587,6 @@ int main(int argc, char **argv) {
     spec_time_extension = 0;
     expire = 0;
     compress = NOT_COMPRESSED;
-    compression_level = 0;
     memset((void *)&repeater, 0, sizeof(repeater));
     srcSpoofing = 0;
     configFile = NULL;
@@ -802,7 +800,7 @@ int main(int argc, char **argv) {
                 if (optarg == NULL) {
                     compress = LZO_COMPRESSED;
                 } else {
-                    compress = ParseCompression(optarg, &compression_level);
+                    compress = ParseCompression(optarg);
                 }
                 if (compress == -1) {
                     LogError("Usage for option -z: set -z=lzo, -z=lz4 or -z=bz2 for valid compression formats");