Merge pull request #10 from ntop/dev

Repo sync

.gitignore

@@ -19,6 +19,8 @@ compile
 /example/.deps/
 /example/Makefile
 /example/ndpiReader
+/example/Makefile.dpdk
+/example/build
 *.in
 /install-sh
 *.ko
@@ -35,7 +37,7 @@ compile
 /m4/ltversion.m4
 /Makefile
 /missing
-*.[oa]
+*.[o]
 *.obj
 *o.cmd
 *o.cmd

CHANGELOG.md

@@ -1,5 +1,119 @@
 # CHANGELOG
 
+#### nDPI 2.6 (December 2018)
+
+## New Supported Protocols and Services
+
+* New Bitcoin, Ethereum, ZCash, Monero dissectors all identified as Mining
+* New Signal.org dissector
+* New Nest Log Sink dissector
+* New UPnP dissector
+* Added support for SMBv1 traffic, split from SMBv23
+
+## Improvements
+
+* Improved Skype detection, merged Skype call in/out into Skype Call
+* Improved heuristics for Skype, Teredo, Netbios
+* Improved SpeedTest (Ookla) detection
+* Improved WhatsApp detection
+* Improved WeChat detection
+* Improved Facebook Messenger detection
+* Improved Messenger/Hangout detection
+* Improved SSL detection, prevent false positives
+* Improved guess for UDP protocols
+* Improved STUN detection
+* Added more Ubuntu servers
+* Added missing categorization with giveup/guess
+* Optimizations for TCP flows that do not start with a SYN packet (early giveup)
+
+## Fixes
+
+* Fixed eDonkey false positives
+* Fixed Dropbox dissector
+* Fixed Spotify dissector
+* Fixed custom protocol loading
+* Fixed missing Application Data packet for TLS
+* Fixed buffer overflows
+* Fixed custom categories match by IP
+* Fixed category field not accounted in ndpi_get_proto_category
+* Fixed null pointer dereference in ndpi_detection_process_packet
+* Fixed compilation on Mac
+
+## Other
+
+* Deb and RPM packages: ndpi with shared libraries and binaries, ndpi-dev with headers and static libraries
+* Protocols now have an optional subprotocol: Spotify cannot have subprotocols, DNS can (DNS.Spotify) 
+* New API functions:
+  - ndpi_fill_ip_protocol_category to handle ICMP flows category
+  - ndpi_flowv4_flow_hash and ndpi_flowv6_flow_hash to support the Community ID Flow Hashing (https://github.com/corelight/community-id-spec)
+  - ndpi_protocol2id to print the protocol as ID
+  - ndpi_get_custom_category_match to search host in custom categories
+* Changed ndpi_detection_giveup API: guess is now part of the call
+* Added DPDK support to ndpiReader
+* Removed Musical.ly protocol (service no longer used)
+* Custom categories have now priority over protocol related categories
+* Improved clang support
+
+------------------------------------------------------------------------
+
+#### nDPI 2.4 (August 2018)
+
+## New Supported Protocols and Services
+
+* Showmax.com
+* Musical.ly
+* RapidVideo
+* VidTO streaming service
+* Apache JServ Protocol
+* Facebook Messenger
+* FacebookZero protocol
+
+## Improvements
+
+* Improved YouTube support
+* Improved Netflix support
+* Updated Google Hangout detection
+* Updated Twitter address range
+* Updated Viber ports, subnet and domain
+* Updated AmazonVideo detection
+* Updated list of FaceBook sites
+* Initial Skype in/out support
+* Improved Tor detection
+* Improved hyperscan support and category definition
+* Custom categories loading, extended ndpiReader (`-c <file>`) for loading name-based categories
+
+## Fixes
+
+* Fixes for Instagram flows classified as Facebook
+* Fixed Spotify detection
+* Fixed minimum packet payload length for SSDP
+* Fixed length check in MSN, x-steam-sid, Tor certificate name
+* Increase client's maximum payload length for SSH
+* Fixed end-of-line bounds handling
+* Fixed substring matching
+* Fix for handling IP address based custom categories
+* Repaired wrong timestamp calculation
+* Fixed memory leak
+* Optimized memory usage
+
+## Other/Changes
+
+* New API calls:
+  * `ndpi_set_detection_preferences()`
+  * `ndpi_load_hostname_category()`
+  * `ndpi_enable_loaded_categories()`
+  * `ndpi_fill_protocol_category()`
+  * `ndpi_process_extra_packet()`
+* Skype CallIn/CallOut are now set as Skype.SkypeCallOut Skype.SkypeCallIn
+* Added support for SMTPS on port 587
+* Changed RTP from VoIP to Media category
+* Added site unavailable category
+* Added custom categories CUSTOM_CATEGORY_MINING, CUSTOM_CATEGORY_MALWARE, CUSTOM_CATEGORY_ADVERTISEMENT, CUSTOM_CATEGORY_BANNED_SITE
+* Implemented hash-based categories
+* Converted some not popular protocols to NDPI_PROTOCOL_GENERIC with category detection
+
+------------------------------------------------------------------------
+
 #### nDPI 2.2.2 (April 2018)
 
 ## Main New Features
@@ -95,7 +209,7 @@
 * Ubiquity AirControl 2
 * HEP (Extensible Encapsulation Protocol)
 * WhatsApp Voice vs WhatsApp (chat, no voice)
-* Viber	
+* Viber
 * Wechat
 * Github
 * Hotmail
@@ -124,7 +238,7 @@
 * Improved HTTP subprotocol matching
 * Implemented DHCP host name extraction
 * Updated Facebook detection by ip server ranges
-* Updated Twitter networks	  
+* Updated Twitter networks
 * Improved Microsoft detection
 * Enhanced Google detection
 * Improved BT-uTP protocol dissection

Makefile.am

@@ -1,8 +1,7 @@
 ACLOCAL_AMFLAGS = -I m4
-
 SUBDIRS = src/lib example tests
 
-pkgconfigdir = $(libdir)/pkgconfig
+pkgconfigdir = $(prefix)/libdata/pkgconfig
 pkgconfig_DATA = libndpi.pc
 
-EXTRA_DIST = libndpi.sym autogen.sh
+EXTRA_DIST = autogen.sh

README.md

@@ -2,6 +2,8 @@
 # nDPI
 
 [![Build Status](https://travis-ci.org/ntop/nDPI.png?branch=dev)](https://travis-ci.org/ntop/nDPI)
+[![Code Quality: Cpp](https://img.shields.io/lgtm/grade/cpp/g/ntop/nDPI.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/ntop/nDPI/context:cpp)
+[![Total Alerts](https://img.shields.io/lgtm/alerts/g/ntop/nDPI.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/ntop/nDPI/alerts)
 
 ## What is nDPI ?
 
@@ -34,15 +36,13 @@ The entire procedure of adding new protocols in detail:
 5. Choose (do not change anything) a selection bitmask from: src/include/ndpi_define.h
 6. Add a new entry in ndpi_set_protocol_detection_bitmask2 in: src/lib/ndpi_main.c
 7. Set protocol default ports in ndpi_init_protocol_defaults in: src/lib/ndpi_main.c
-8. Add the new protocol file to: src/lib/Makefile.am
-9.  ./autogen.sh
-10. ./configure
-11. make
-12. make check
+8.  ./autogen.sh
+9. make
+10. make check
 
 ### How to use nDPI to Block Selected Traffic
 
-You can use nDPI to selectively block selected Internet traffic by embedding it onto an application (remember that nDPI us just a library). Both [ntopng](https://github.com/ntop/ntopng) and [nProbe cento](http://www.ntop.org/products/netflow/nprobe-cento/) can do this.
+You can use nDPI to selectively block selected Internet traffic by embedding it onto an application (remember that nDPI is just a library). Both [ntopng](https://github.com/ntop/ntopng) and [nProbe cento](http://www.ntop.org/products/netflow/nprobe-cento/) can do this.
 
 ### DISCLAIMER
 While we do our best to detect network protocols, we cannot guarantee that our software is error free and 100% accurate in protocol detection. Please make sure that you respect the privacy of users and you have proper authorization to listen, capture and inspect network traffic.

autogen.sh

@@ -1,11 +1,11 @@
 #!/bin/sh
 
 NDPI_MAJOR="2"
-NDPI_MINOR="3"
+NDPI_MINOR="5"
 NDPI_PATCH="0"
 NDPI_VERSION_SHORT="$NDPI_MAJOR.$NDPI_MINOR.$NDPI_PATCH"
 
-rm -f configure config.h config.h.in src/lib/Makefile.in
+rm -f configure config.h config.h.in
 
 AUTOCONF=$(command -v autoconf)
 AUTOMAKE=$(command -v automake)

configure.seed

@@ -10,6 +10,7 @@ AC_PROG_CC
 AM_PROG_CC_C_O
 AX_PTHREAD
 
+NDPI_VERSION_SHORT="@NDPI_VERSION_SHORT@"
 NDPI_MAJOR="@NDPI_MAJOR@"
 NDPI_MINOR="@NDPI_MINOR@"
 NDPI_PATCH="@NDPI_PATCH@"
@@ -39,6 +40,14 @@ AC_CHECK_HEADERS([netinet/in.h stdint.h stdlib.h string.h unistd.h])
 
 PCAP_HOME=$HOME/PF_RING/userland
 
+DPDK_TARGET=
+if test -d $HOME/DPDK; then :
+     echo "Enabling DPDK support in ndpiReader"
+     DPDK_TARGET=dpdk
+else
+    echo  "DPDK support disabled (missing $HOME/DPDK)"
+fi
+
 if test -d $PCAP_HOME; then :
      echo -n ""
 else
@@ -51,19 +60,38 @@ else
      AC_CHECK_LIB([numa], [numa_available], [LIBNUMA="-lnuma"])
 fi
 
-
+MACHINE=`uname -m`
+SYSTEM=`uname -s`
+if test $SYSTEM = "Darwin"; then
+ CC=clang
+fi
+
+if test $ax_cv_PTHREAD_CLANG = "yes"; then
+ CC=clang
+fi
+
 HS_LIB=
 HS_INC=
-BKP=$LIBS
-LIBS="$LIBS -lstdc++ -lm"
-AC_CHECK_LIB([hs], [hs_compile_multi], AC_DEFINE_UNQUOTED(HAVE_HYPERSCAN, 1, [Intel Hyperscan is present]))
-LIBS=$BKP
-
-if test "x$ac_cv_lib_hs_hs_compile_multi" = xyes; then :
-AC_CHECK_LIB([m], [pow])
-AC_CHECK_LIB([stdc++], [main])
-HS_INC=`pkg-config --cflags libhs`
-HS_LIB=`pkg-config --libs libhs`
+
+AC_ARG_WITH(hyperscan,          [  --with-hyperscan        Enable nDPI build with Intel Hyperscan])
+
+if test "${with_hyperscan+set}" = set; then
+     BKP=$LIBS
+     LIBS="$LIBS -lstdc++ -lm"
+     AC_CHECK_LIB([hs], [hs_compile_multi], AC_DEFINE_UNQUOTED(HAVE_HYPERSCAN, 1, [Intel Hyperscan is present]))
+     LIBS=$BKP
+
+     if test "x$ac_cv_lib_hs_hs_compile_multi" = xyes; then :
+          AC_CHECK_LIB([m], [pow])
+          AC_CHECK_LIB([stdc++], [main])
+          HS_INC=`pkg-config --cflags libhs`
+          HS_LIB=`pkg-config --libs libhs`
+          LDFLAGS="$LDFLAGS $HS_LIB"
+	  AC_MSG_RESULT([compiling with Intel Hyperscan])
+     else
+          AC_MSG_RESULT([Intel Hyperscan not found, exiting. See https://github.com/intel/hyperscan/blob/master/doc/dev-reference/getting_started.rst for install/build instructions])
+          exit 1
+     fi
 fi
 
 if test -f $PCAP_HOME/libpcap/libpcap.a; then :
@@ -119,19 +147,21 @@ AC_ARG_ENABLE([debug-messages],
 
 AC_CHECK_LIB(pthread, pthread_setaffinity_np, AC_DEFINE_UNQUOTED(HAVE_PTHREAD_SETAFFINITY_NP, 1, [libc has pthread_setaffinity_np]))
 
-AC_CONFIG_FILES([Makefile src/lib/Makefile example/Makefile tests/Makefile libndpi.pc src/include/ndpi_define.h])
+AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile])
 AC_CONFIG_HEADERS(src/include/ndpi_config.h)
 AC_SUBST(GIT_RELEASE)
 AC_SUBST(NDPI_MAJOR)
 AC_SUBST(NDPI_MINOR)
 AC_SUBST(NDPI_PATCH)
+AC_SUBST(NDPI_VERSION_SHORT)
 AC_SUBST(SVN_DATE)
 AC_SUBST(JSON_C_LIB)
 AC_SUBST(PCAP_INC)
 AC_SUBST(PCAP_LIB)
 AC_SUBST(DL_LIB)
 AC_SUBST(HS_LIB)
 AC_SUBST(HS_INC)
+AC_SUBST(DPDK_TARGET)
 AC_SUBST(HAVE_PTHREAD_SETAFFINITY_NP)
 
 AC_OUTPUT

example/Makefile.dpdk.in

@@ -0,0 +1,27 @@
+#
+# Run 'make -f Makefile.dpdk' to compile the DPDK examples
+#
+# See http://core.dpdk.org/doc/quick-start/ for DPDK installation and setup
+#
+ifeq ($(RTE_SDK),)
+#$(error "Please define RTE_SDK environment variable")
+RTE_SDK = $(HOME)/DPDK
+RTE_TARGET = build
+endif
+
+# Default target, can be overridden by command line or environment
+RTE_TARGET ?= x86_64-native-linuxapp-gcc
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+APP = ndpiReader.dpdk
+LIBNDPI = $(PWD)/../src/lib/libndpi.a
+
+SRCS-y := ndpi_util.c ndpiReader.c
+
+CFLAGS += -g
+CFLAGS += -Wno-strict-prototypes -Wno-missing-prototypes -Wno-missing-declarations -Wno-unused-parameter -I $(PWD)/../src/include @CFLAGS@ -DUSE_DPDK
+LDLIBS = $(LIBNDPI) @PCAP_LIB@ -lpthread @LDFLAGS@
+
+include $(RTE_SDK)/mk/rte.extapp.mk
+

example/Makefile.in

@@ -0,0 +1,33 @@
+CC=@CC@
+CFLAGS=-g -I../src/include @CFLAGS@
+LIBNDPI=../src/lib/libndpi.a
+LDFLAGS=$(LIBNDPI) @PCAP_LIB@ -lpthread @LDFLAGS@
+OBJS=ndpiReader.o ndpi_util.o
+PREFIX?=/usr/local
+
+all: ndpiReader @DPDK_TARGET@
+
+ndpiReader: $(OBJS) $(LIBNDPI)
+	$(CXX) $(CFLAGS) $(OBJS) -o $@ $(LDFLAGS)
+
+%.o: %.c $(HEADERS) Makefile
+	$(CC) $(CFLAGS) -c $< -o $@
+
+install:
+	mkdir -p $(DESTDIR)$(PREFIX)/bin/
+	cp ndpiReader $(DESTDIR)$(PREFIX)/bin/
+	[ -f build/app/ndpiReader.dpdk ] && cp build/app/ndpiReader.dpdk $(DESTDIR)$(PREFIX)/bin/ || true
+	[ -f ndpiReader.dpdk ] && cp ndpiReader.dpdk $(DESTDIR)$(PREFIX)/bin/ || true
+
+dpdk:
+	make -f Makefile.dpdk
+
+clean:
+	/bin/rm -f *.o ndpiReader ndpiReader.dpdk
+	/bin/rm -f .*.dpdk.cmd .*.o.cmd *.dpdk.map .*.o.d
+	/bin/rm -f _install _postbuild _postinstall _preinstall
+	/bin/rm -rf build
+
+distclean: clean
+	/bin/rm -f Makefile.dpdk
+	/bin/rm -f Makefile