Added support for roaring bitmap v3 (#2355)

* Integrated RoaringBitmap v3

* Renamed ndpi_bitmap64 ro ndpi_bitmap64_fuse

* Fixes to ndpi_bitmap for new roaring library

* Fixes for bitmap serialization

* Fixed format

* Warning fix

* Conversion fix

* Warning fix

* Added check for roaring v3 support

* Updated file name

* Updated path

* Uses clang-9 (instead of clang-7) for builds

* Fixed fuzz_ds_bitmap64_fuse

* Fixes nDPI printf handling

* Disabled printf

* Yet another printf fix

* Cleaup

* Fx for compiling on older platforms

* Fixes for old compilers

* Initialization changes

* Added compiler check

* Fixes for old compilers

* Inline function is not static inline

* Added missing include

.github/workflows/build.yml

@@ -107,7 +107,7 @@ jobs:
             msan: "--with-sanitizer"
             nBPF: ""
             lto_gold_linker: "--with-lto-and-gold-linker"
-          - compiler: "clang-7" # "Oldest" clang easily available
+          - compiler: "clang-9" # "Oldest" clang easily available
             os: ubuntu-20.04
             arch: "x86_64"
             gcrypt: ""

configure.ac

@@ -120,6 +120,25 @@ AC_CHECK_TOOL(AR, ar, [false])
 
 AC_LANG_WERROR
 
+AC_MSG_CHECKING([whether roaring v3 works])
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
+	    #if __has_include (<stdio.h>)
+	    #endif
+	    ]])],
+	    [
+	      AC_MSG_RESULT([yes])
+	      ],
+	      [
+	        AC_MSG_RESULT([no])
+		  AC_DEFINE_UNQUOTED(USE_ROARING_V2, "1", [Use CRoaring 2.1.x])
+  ])
+
+GCC_VERSION=`gcc --version | cut -d ' ' -f 3 | head -1|cut -d '.' -f 1`
+if [ "${GCC_VERSION}" -lt "7" ]; then
+ AC_DEFINE_UNQUOTED(USE_ROARING_V2, "1", [Use CRoaring 2.1.x])
+fi
+
+
 NDPI_MAJOR=`echo "${PACKAGE_VERSION}" | cut -d . -f 1`
 NDPI_MINOR=`echo "${PACKAGE_VERSION}" | cut -d . -f 2`
 NDPI_PATCH=`echo "${PACKAGE_VERSION}" | cut -d . -f 3`

example/ndpiReader.c

@@ -5605,7 +5605,7 @@ void compressedBitmapUnitTest() {
   size_t ser;
   char *buf;
   ndpi_bitmap_iterator *it;
-  u_int32_t value;
+  u_int64_t value;
 
   for(i=0; i<1000; i++) {
     u_int32_t v = rand();
@@ -5621,12 +5621,12 @@ void compressedBitmapUnitTest() {
   assert(ser > 0);
 
   if(trace) printf("len: %u\n", (unsigned int)ser);
-  b1 = ndpi_bitmap_deserialize(buf);
+  b1 = ndpi_bitmap_deserialize(buf, ser);
   assert(b1);
 
   assert((it = ndpi_bitmap_iterator_alloc(b)));
   while(ndpi_bitmap_iterator_next(it, &value)) {
-    if(trace) printf("%u ", value);
+    if(trace) printf("%lu ", (unsigned long)value);
   }
 
   if(trace) printf("\n");

fuzz/Makefile.am

@@ -2,7 +2,7 @@ bin_PROGRAMS = fuzz_process_packet fuzz_ndpi_reader fuzz_ndpi_reader_alloc_fail
 #Alghoritms
 bin_PROGRAMS += fuzz_alg_bins fuzz_alg_hll fuzz_alg_hw_rsi_outliers_da fuzz_alg_jitter fuzz_alg_ses_des fuzz_alg_crc32_md5 fuzz_alg_bytestream
 #Data structures
-bin_PROGRAMS += fuzz_ds_patricia fuzz_ds_ahocorasick fuzz_ds_libcache fuzz_ds_tree fuzz_ds_ptree fuzz_ds_hash fuzz_ds_cmsketch fuzz_ds_bitmap64 fuzz_ds_domain_classify
+bin_PROGRAMS += fuzz_ds_patricia fuzz_ds_ahocorasick fuzz_ds_libcache fuzz_ds_tree fuzz_ds_ptree fuzz_ds_hash fuzz_ds_cmsketch fuzz_ds_bitmap64_fuse fuzz_ds_domain_classify
 #Third party
 bin_PROGRAMS += fuzz_libinjection fuzz_binaryfusefilter
 #Internal crypto
@@ -328,20 +328,20 @@ fuzz_ds_cmsketch_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
     $(LIBTOOLFLAGS) --mode=link $(CXX) @NDPI_CFLAGS@ $(AM_CXXFLAGS) $(CXXFLAGS) \
     $(fuzz_ds_cmsketch_LDFLAGS) @NDPI_LDFLAGS@ $(LDFLAGS) -o $@
 
-fuzz_ds_bitmap64_SOURCES = fuzz_ds_bitmap64.cpp fuzz_common_code.c
-fuzz_ds_bitmap64_CXXFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS)
-fuzz_ds_bitmap64_CFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS)
-fuzz_ds_bitmap64_LDADD = ../src/lib/libndpi.a $(ADDITIONAL_LIBS)
-fuzz_ds_bitmap64_LDFLAGS = $(LIBS)
+fuzz_ds_bitmap64_fuse_SOURCES = fuzz_ds_bitmap64_fuse.cpp fuzz_common_code.c
+fuzz_ds_bitmap64_fuse_CXXFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS)
+fuzz_ds_bitmap64_fuse_CFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS)
+fuzz_ds_bitmap64_fuse_LDADD = ../src/lib/libndpi.a $(ADDITIONAL_LIBS)
+fuzz_ds_bitmap64_fuse_LDFLAGS = $(LIBS)
 if HAS_FUZZLDFLAGS
-fuzz_ds_bitmap64_CXXFLAGS += $(LIB_FUZZING_ENGINE)
-fuzz_ds_bitmap64_CFLAGS += $(LIB_FUZZING_ENGINE)
-fuzz_ds_bitmap64_LDFLAGS += $(LIB_FUZZING_ENGINE)
+fuzz_ds_bitmap64_fuse_CXXFLAGS += $(LIB_FUZZING_ENGINE)
+fuzz_ds_bitmap64_fuse_CFLAGS += $(LIB_FUZZING_ENGINE)
+fuzz_ds_bitmap64_fuse_LDFLAGS += $(LIB_FUZZING_ENGINE)
 endif
 # force usage of CXX for linker
-fuzz_ds_bitmap64_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+fuzz_ds_bitmap64_fuse_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
     $(LIBTOOLFLAGS) --mode=link $(CXX) @NDPI_CFLAGS@ $(AM_CXXFLAGS) $(CXXFLAGS) \
-    $(fuzz_ds_bitmap64_LDFLAGS) @NDPI_LDFLAGS@ $(LDFLAGS) -o $@
+    $(fuzz_ds_bitmap64_fuse_LDFLAGS) @NDPI_LDFLAGS@ $(LDFLAGS) -o $@
 
 fuzz_ds_domain_classify_SOURCES = fuzz_ds_domain_classify.cpp fuzz_common_code.c
 fuzz_ds_domain_classify_CXXFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS)
@@ -729,15 +729,15 @@ files_corpus_fuzz_readerutils_parseprotolist :=  $(wildcard corpus/fuzz_readerut
 fuzz_readerutils_parseprotolist_seed_corpus.zip: $(files_corpus_fuzz_readerutils_parseprotolist)
 	zip -j fuzz_readerutils_parseprotolist_seed_corpus.zip $(files_corpus_fuzz_readerutils_parseprotolist)
 
-files_corpus_fuzz_ds_bitmap64 :=  $(wildcard corpus/fuzz_ds_bitmap64/*)
-fuzz_ds_bitmap64_seed_corpus.zip: $(files_corpus_fuzz_ds_bitmap64)
-	zip -j fuzz_ds_bitmap64_seed_corpus.zip $(files_corpus_fuzz_ds_bitmap64)
+files_corpus_fuzz_ds_bitmap64_fuse :=  $(wildcard corpus/fuzz_ds_bitmap64_fuse/*)
+fuzz_ds_bitmap64_fuse_seed_corpus.zip: $(files_corpus_fuzz_ds_bitmap64_fuse)
+	zip -j fuzz_ds_bitmap64_fuse_seed_corpus.zip $(files_corpus_fuzz_ds_bitmap64_fuse)
 
 files_corpus_fuzz_ds_domain_classify :=  $(wildcard corpus/fuzz_ds_domain_classify/*)
 fuzz_ds_domain_classify_seed_corpus.zip: $(files_corpus_fuzz_ds_domain_classify)
 	zip -j fuzz_ds_domain_classify_seed_corpus.zip $(files_corpus_fuzz_ds_domain_classify)
 
-corpus: fuzz_ndpi_reader_seed_corpus.zip fuzz_ndpi_reader_alloc_fail_seed_corpus.zip fuzz_ndpi_reader_payload_analyzer_seed_corpus.zip fuzz_quic_get_crypto_data_seed_corpus.zip fuzz_alg_ses_des_seed_corpus.zip fuzz_alg_bins_seed_corpus.zip fuzz_alg_hll_seed_corpus.zip fuzz_alg_jitter_seed_corpus.zip fuzz_ds_libcache_seed_corpus.zip fuzz_community_id_seed_corpus.zip fuzz_serialization_seed_corpus.zip fuzz_ds_ptree_seed_corpus.zip fuzz_alg_crc32_md5_seed_corpus.zip fuzz_alg_bytestream_seed_corpus.zip fuzz_libinjection_seed_corpus.zip fuzz_tls_certificate_seed_corpus.zip fuzz_filecfg_protocols_seed_corpus.zip fuzz_readerutils_workflow_seed_corpus.zip fuzz_readerutils_parseprotolist_seed_corpus.zip fuzz_ds_bitmap64_seed_corpus.zip fuzz_ds_domain_classify_seed_corpus.zip fuzz_filecfg_protocols_seed_corpus.zip fuzz_filecfg_categories_seed_corpus.zip fuzz_filecfg_config_seed_corpus.zip fuzz_filecfg_category_seed_corpus.zip fuzz_is_stun_udp_seed_corpus.zip fuzz_is_stun_tcp_seed_corpus.zip fuzz_filecfg_malicious_sha1_seed_corpus.zip fuzz_filecfg_malicious_ja3_seed_corpus.zip fuzz_filecfg_risk_domains_seed_corpus.zip
+corpus: fuzz_ndpi_reader_seed_corpus.zip fuzz_ndpi_reader_alloc_fail_seed_corpus.zip fuzz_ndpi_reader_payload_analyzer_seed_corpus.zip fuzz_quic_get_crypto_data_seed_corpus.zip fuzz_alg_ses_des_seed_corpus.zip fuzz_alg_bins_seed_corpus.zip fuzz_alg_hll_seed_corpus.zip fuzz_alg_jitter_seed_corpus.zip fuzz_ds_libcache_seed_corpus.zip fuzz_community_id_seed_corpus.zip fuzz_serialization_seed_corpus.zip fuzz_ds_ptree_seed_corpus.zip fuzz_alg_crc32_md5_seed_corpus.zip fuzz_alg_bytestream_seed_corpus.zip fuzz_libinjection_seed_corpus.zip fuzz_tls_certificate_seed_corpus.zip fuzz_filecfg_protocols_seed_corpus.zip fuzz_readerutils_workflow_seed_corpus.zip fuzz_readerutils_parseprotolist_seed_corpus.zip fuzz_ds_bitmap64_fuse_seed_corpus.zip fuzz_ds_domain_classify_seed_corpus.zip fuzz_filecfg_protocols_seed_corpus.zip fuzz_filecfg_categories_seed_corpus.zip fuzz_filecfg_config_seed_corpus.zip fuzz_filecfg_category_seed_corpus.zip fuzz_is_stun_udp_seed_corpus.zip fuzz_is_stun_tcp_seed_corpus.zip fuzz_filecfg_malicious_sha1_seed_corpus.zip fuzz_filecfg_malicious_ja3_seed_corpus.zip fuzz_filecfg_risk_domains_seed_corpus.zip
 	cp corpus/fuzz_*seed_corpus.zip .
 
 #Create dictionaries exactly as expected by oss-fuzz.
@@ -787,7 +787,7 @@ distdir:
 		-o -path './corpus/fuzz_alg_crc32_md5/*' \
 		-o -path './corpus/fuzz_alg_bytestream/*' \
 		-o -path './corpus/fuzz_ds_libcache/*' \
-		-o -path './corpus/fuzz_ds_bitmap64/*' \
+		-o -path './corpus/fuzz_ds_bitmap64_fuse/*' \
 		-o -path './corpus/fuzz_ds_domain_classify/*' \
 		-o -path './corpus/fuzz_ds_ptree/*' | xargs -I'{}' cp -r '{}' '$(distdir)/{}'
 

fuzz/fuzz_ds_bitmap64.cpp → fuzz/fuzz_ds_bitmap64_fuse.cpp

@@ -7,23 +7,23 @@
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   FuzzedDataProvider fuzzed_data(data, size);
   u_int16_t i, num_iteration, is_added = 0;
-  ndpi_bitmap64 *b;
+  ndpi_bitmap64_fuse *b;
   bool rc;
   u_int64_t value, value_added;
 
   /* To allow memory allocation failures */
   fuzz_set_alloc_callbacks_and_seed(size);
 
-  b = ndpi_bitmap64_alloc();
+  b = ndpi_bitmap64_fuse_alloc();
 
   if(fuzzed_data.ConsumeBool())
-    ndpi_bitmap64_compress(b);
+    ndpi_bitmap64_fuse_compress(b);
 
   num_iteration = fuzzed_data.ConsumeIntegral<u_int16_t>();
   for (i = 0; i < num_iteration; i++) {
     value = fuzzed_data.ConsumeIntegral<u_int64_t>();
 
-    rc = ndpi_bitmap64_set(b, value);
+    rc = ndpi_bitmap64_fuse_set(b, value);
     /* Keep one random entry really added */
     if (rc == true && is_added == 0 && fuzzed_data.ConsumeBool()) {
       value_added = value;
@@ -32,23 +32,23 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   }
 
   if(fuzzed_data.ConsumeBool())
-    ndpi_bitmap64_compress(b);
+    ndpi_bitmap64_fuse_compress(b);
 
   /* "Random" search */
   num_iteration = fuzzed_data.ConsumeIntegral<u_int8_t>();
   for (i = 0; i < num_iteration; i++) {
     value = fuzzed_data.ConsumeIntegral<u_int64_t>();
 
-    ndpi_bitmap64_isset(b, value);
+    ndpi_bitmap64_fuse_isset(b, value);
   }
   /* Search of an added entry */
   if (is_added) {
-    ndpi_bitmap64_isset(b, value_added);
+    ndpi_bitmap64_fuse_isset(b, value_added);
   }
 
-  ndpi_bitmap64_size(b);
+  ndpi_bitmap64_fuse_size(b);
 
-  ndpi_bitmap64_free(b);
+  ndpi_bitmap64_fuse_free(b);
 
   return 0;
 }

src/include/ndpi_api.h

@@ -2012,18 +2012,16 @@ extern "C" {
   */
 
   ndpi_bitmap* ndpi_bitmap_alloc(void);
-  ndpi_bitmap* ndpi_bitmap_alloc_size(u_int32_t size);
   void ndpi_bitmap_free(ndpi_bitmap* b);
   ndpi_bitmap* ndpi_bitmap_copy(ndpi_bitmap* b);
   u_int64_t ndpi_bitmap_cardinality(ndpi_bitmap* b);
   bool ndpi_bitmap_is_empty(ndpi_bitmap* b);
-  void ndpi_bitmap_set(ndpi_bitmap* b, u_int32_t value);
-  void ndpi_bitmap_unset(ndpi_bitmap* b, u_int32_t value);
-  bool ndpi_bitmap_isset(ndpi_bitmap* b, u_int32_t value);
-  void ndpi_bitmap_clear(ndpi_bitmap* b);
+  void ndpi_bitmap_set(ndpi_bitmap* b, u_int64_t value);
+  void ndpi_bitmap_unset(ndpi_bitmap* b, u_int64_t value);
+  bool ndpi_bitmap_isset(ndpi_bitmap* b, u_int64_t value);
 
   size_t ndpi_bitmap_serialize(ndpi_bitmap* b, char **buf);
-  ndpi_bitmap* ndpi_bitmap_deserialize(char *buf);
+  ndpi_bitmap* ndpi_bitmap_deserialize(char *buf, size_t buf_len);
 
   void ndpi_bitmap_and(ndpi_bitmap* a, ndpi_bitmap* b_and);
   ndpi_bitmap* ndpi_bitmap_and_alloc(ndpi_bitmap* a, ndpi_bitmap* b_and);
@@ -2035,7 +2033,7 @@ extern "C" {
 
   ndpi_bitmap_iterator* ndpi_bitmap_iterator_alloc(ndpi_bitmap* b);
   void ndpi_bitmap_iterator_free(ndpi_bitmap* b);
-  bool ndpi_bitmap_iterator_next(ndpi_bitmap_iterator* i, u_int32_t *value);
+  bool ndpi_bitmap_iterator_next(ndpi_bitmap_iterator* i, u_int64_t *value);
 
   /* ******************************* */
 
@@ -2045,16 +2043,16 @@ extern "C" {
 
     This is
     - a probabilistic datastructure !!! (i.e. be prepared to false positives)
-    - immutable (i.e. adding keys after a search (i.e. ndpi_bitmap64_isset)
+    - immutable (i.e. adding keys after a search (i.e. ndpi_bitmap64_fuse_isset)
       is not allowed
    */
 
-  ndpi_bitmap64* ndpi_bitmap64_alloc(void);
-  bool ndpi_bitmap64_set(ndpi_bitmap64 *b, u_int64_t value);
-  bool ndpi_bitmap64_compress(ndpi_bitmap64 *b);
-  bool ndpi_bitmap64_isset(ndpi_bitmap64 *b, u_int64_t value);
-  void ndpi_bitmap64_free(ndpi_bitmap64 *b);
-  u_int32_t ndpi_bitmap64_size(ndpi_bitmap64 *b);
+  ndpi_bitmap64_fuse* ndpi_bitmap64_fuse_alloc(void);
+  bool ndpi_bitmap64_fuse_set(ndpi_bitmap64_fuse *b, u_int64_t value);
+  bool ndpi_bitmap64_fuse_compress(ndpi_bitmap64_fuse *b);
+  bool ndpi_bitmap64_fuse_isset(ndpi_bitmap64_fuse *b, u_int64_t value);
+  void ndpi_bitmap64_fuse_free(ndpi_bitmap64_fuse *b);
+  u_int32_t ndpi_bitmap64_fuse_size(ndpi_bitmap64_fuse *b);
 
   /* ******************************* */
 

src/include/ndpi_define.h.in

@@ -176,6 +176,7 @@
 #define get_u_int32_t(X,O)  (*(u_int32_t *)((&(((u_int8_t *)X)[O]))))
 #if defined(__arm__)
 #include <stdint.h>
+#include <string.h>
 static inline uint64_t get_u_int64_t(const uint8_t* X, int O)
 {
   uint64_t tmp;

src/include/ndpi_typedefs.h

@@ -1141,6 +1141,7 @@ typedef struct ndpi_proto {
 
 typedef void ndpi_bitmap;
 typedef void ndpi_bitmap64;
+typedef void ndpi_bitmap64_fuse; /* probabilistic */
 typedef void ndpi_bitmap_iterator;
 typedef void ndpi_filter;
 
@@ -1155,7 +1156,7 @@ typedef struct {
 typedef struct {
   struct {
     u_int16_t class_id;
-    ndpi_bitmap64 *domains;
+    ndpi_bitmap64_fuse *domains;
   } classes[MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS];
 } ndpi_domain_classify;