Add support for flow client/server information (#1671)

In a lot of places in ndPI we use *packet* source/dest info
(address/port/direction) when we are interested in *flow* client/server
info, instead.

Add basic logic to autodetect this kind of information.

nDPI doesn't perform any "flow management" itself but this task is
delegated to the external application. It is then likely that the
application might provide more reliable hints about flow
client/server direction and about the TCP handshake presence: in that case,
these information might be (optionally) passed to the library, disabling
the internal "autodetect" logic.

These new fields have been used in some LRU caches and in the "guessing"
algorithm.
It is quite likely that some other code needs to be updated.

example/ndpiSimpleIntegration.c

@@ -884,7 +884,7 @@ static void ndpi_process_packet(uint8_t * const args,
   flow_to_process->detected_l7_protocol =
     ndpi_detection_process_packet(workflow->ndpi_struct, flow_to_process->ndpi_flow,
 				  ip != NULL ? (uint8_t *)ip : (uint8_t *)ip6,
-				  ip_size, time_ms);
+				  ip_size, time_ms, NULL);
 
   if (ndpi_is_protocol_detected(workflow->ndpi_struct,
 				flow_to_process->detected_l7_protocol) != 0 &&

example/reader_util.c

@@ -1543,6 +1543,8 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
   }
 
   if(!flow->detection_completed) {
+    struct ndpi_flow_input_info input_info;
+
     u_int enough_packets =
       (((proto == IPPROTO_UDP) && ((flow->src2dst_packets + flow->dst2src_packets) > max_num_udp_dissected_pkts))
        || ((proto == IPPROTO_TCP) && ((flow->src2dst_packets + flow->dst2src_packets) > max_num_tcp_dissected_pkts))) ? 1 : 0;
@@ -1558,9 +1560,13 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
     else
       workflow->stats.dpi_packet_count[2]++;
 
+    memset(&input_info, '\0', sizeof(input_info)); /* To be sure to set to "unknown" any fields */
+    /* Set here any information (easily) available; in this trivial example we don't have any */
+    input_info.in_pkt_dir = NDPI_IN_PKT_DIR_UNKNOWN;
+    input_info.seen_flow_beginning = NDPI_FLOW_BEGINNING_UNKNOWN;
     flow->detected_protocol = ndpi_detection_process_packet(workflow->ndpi_struct, ndpi_flow,
 							    iph ? (uint8_t *)iph : (uint8_t *)iph6,
-							    ipsize, time_ms);
+							    ipsize, time_ms, &input_info);
 
     enough_packets |= ndpi_flow->fail_with_unknown;
     if(enough_packets || (flow->detected_protocol.app_protocol != NDPI_PROTOCOL_UNKNOWN)) {

fuzz/fuzz_process_packet.c

@@ -26,7 +26,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   struct ndpi_flow_struct *ndpi_flow = ndpi_flow_malloc(SIZEOF_FLOW_STRUCT);
   memset(ndpi_flow, 0, SIZEOF_FLOW_STRUCT);
   ndpi_protocol detected_protocol =
-    ndpi_detection_process_packet(ndpi_info_mod, ndpi_flow, Data, Size, 0);
+    ndpi_detection_process_packet(ndpi_info_mod, ndpi_flow, Data, Size, 0, NULL);
   ndpi_protocol guessed_protocol =
     ndpi_detection_giveup(ndpi_info_mod, ndpi_flow, 1, &protocol_was_guessed);
 

python/ndpi/ndpi.py

@@ -45,12 +45,13 @@ def api_version(self):
     def revision(self):
         return ffi.string(lib.ndpi_revision()).decode('utf-8', errors='ignore')
 
-    def process_packet(self, flow, packet, packet_time_ms):
+    def process_packet(self, flow, packet, packet_time_ms, input_info):
         p = lib.ndpi_detection_process_packet(self._detection_module,
                                               flow.C,
                                               packet,
                                               len(packet),
-                                              int(packet_time_ms))
+                                              int(packet_time_ms),
+                                              input_info)
         return ndpi_protocol(C=p,
                              master_protocol=p.master_protocol,
                              app_protocol=p.app_protocol,

python/ndpi/ndpi_build.py

@@ -56,7 +56,8 @@
                                             struct ndpi_flow_struct *flow,
                                             const unsigned char *packet,
                                             const unsigned short packetlen,
-                                            const u_int64_t packet_time_ms);
+                                            const u_int64_t packet_time_ms,
+                                            const struct ndpi_flow_input_info *input_info);
 ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_struct,
                                     struct ndpi_flow_struct *flow,
                                     u_int8_t enable_guess,

python/ndpi_example.py

@@ -14,7 +14,7 @@
 """
 
 from collections import namedtuple
-from ndpi import NDPI, NDPIFlow
+from ndpi import NDPI, NDPIFlow, ffi
 import argparse
 import socket
 import dpkt
@@ -131,15 +131,15 @@ def parse_arguments():
                 key = ppkt_to_flow_key(ppkt)
                 try:  # Try a Flow update
                     flow = flow_cache[key]
-                    flow.detected_protocol = nDPI.process_packet(flow.ndpi_flow, ppkt.ip_bytes, time_ms)
+                    flow.detected_protocol = nDPI.process_packet(flow.ndpi_flow, ppkt.ip_bytes, time_ms, ffi.NULL)
                     flow.pkts += 1
                     flow.bytes += len(packet)
                 except KeyError:  # New Flow
                     flow = Flow()
                     flow.index = flow_count
                     flow_count += 1
                     flow.ndpi_flow = NDPIFlow()  # We create an nDPIFlow object per Flow
-                    flow.detected_protocol = nDPI.process_packet(flow.ndpi_flow, ppkt.ip_bytes, time_ms)
+                    flow.detected_protocol = nDPI.process_packet(flow.ndpi_flow, ppkt.ip_bytes, time_ms, ffi.NULL)
                     flow.pkts += 1
                     flow.bytes += len(packet)
                     flow_cache[key] = flow

python/tests.py

@@ -13,15 +13,15 @@
 ------------------------------------------------------------------------------------------------------------------------
 """
 
-from ndpi import NDPI, NDPIFlow
+from ndpi import NDPI, NDPIFlow, ffi
 import time
 
 
 if __name__ == '__main__':
     try:
         nDPI = NDPI()
         ndpi_flow = NDPIFlow()
-        nDPI.process_packet(ndpi_flow, b'', time.time())
+        nDPI.process_packet(ndpi_flow, b'', time.time(), ffi.NULL)
         nDPI.giveup(ndpi_flow)
         print("nDPI Python bindings: OK")
     except Exception:

src/include/ndpi_api.h.in

@@ -302,14 +302,16 @@ extern "C" {
    * @par    packet         = unsigned char pointer to the Layer 3 (IP header)
    * @par    packetlen      = the length of the packet
    * @par    packet_time_ms = the current timestamp for the packet (expressed in msec)
+   * @par    input_info     = (optional) flow information provided by the (external) flow manager
    * @return void
    *
    */
   void ndpi_process_extra_packet(struct ndpi_detection_module_struct *ndpi_struct,
 				 struct ndpi_flow_struct *flow,
 				 const unsigned char *packet,
 				 const unsigned short packetlen,
-				 const u_int64_t packet_time_ms);
+				 const u_int64_t packet_time_ms,
+				 const struct ndpi_flow_input_info *input_info);
 
   /**
    * Processes one packet and returns the ID of the detected protocol.
@@ -320,14 +322,16 @@ extern "C" {
    * @par    packet         = unsigned char pointer to the Layer 3 (IP header)
    * @par    packetlen      = the length of the packet
    * @par    packet_time_ms = the current timestamp for the packet (expressed in msec)
+   * @par    input_info     = (optional) flow information provided by the (external) flow manager
    * @return the detected ID of the protocol
    *
    */
   ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct *ndpi_struct,
 					      struct ndpi_flow_struct *flow,
 					      const unsigned char *packet,
 					      const unsigned short packetlen,
-					      const u_int64_t packet_time_ms);
+					      const u_int64_t packet_time_ms,
+					      const struct ndpi_flow_input_info *input_info);
   /**
    * Get the main protocol of the passed flows for the detected module
    *

src/include/ndpi_main.h

@@ -167,6 +167,10 @@ extern "C" {
 
   int64_t ndpi_asn1_ber_decode_length(const unsigned char *payload, int payload_len, u_int16_t *value_len);
 
+  int ndpi_current_pkt_from_client_to_server(const struct ndpi_packet_struct *packet, const struct ndpi_flow_struct *flow);
+  int ndpi_current_pkt_from_server_to_client(const struct ndpi_packet_struct *packet, const struct ndpi_flow_struct *flow);
+  int ndpi_seen_flow_beginning(const struct ndpi_flow_struct *flow);
+
 #ifdef __cplusplus
 }
 #endif

src/include/ndpi_typedefs.h

@@ -528,6 +528,31 @@ struct ndpi_vxlanhdr {
 } PACK_OFF;
 
 /* ************************************************************ */
+
+/**
+ * The application might inform the library about client/server direction
+ */
+#define NDPI_IN_PKT_DIR_UNKNOWN		0	/**< The application doesn't provide this kind of information */
+#define NDPI_IN_PKT_DIR_C_TO_S		1	/**< Current packet is from client to server */
+#define NDPI_IN_PKT_DIR_S_TO_C		2	/**< Current packet is from server to client */
+
+/**
+ * The application might choose to not pass TCP handshake packets to the library
+ * (for performance reasons), but it might want to inform the library itlsef that these
+ * packets have been captured/seen anyway (to avoid losing classifiation capabilities).
+ */
+#define NDPI_FLOW_BEGINNING_UNKNOWN	0	/**< The application doesn't provide this kind of information */
+#define NDPI_FLOW_BEGINNING_SEEN	1	/**< The application informs the library that the TCP handshake has been seen (even if its packets might not have been passed to the library) */
+#define NDPI_FLOW_BEGINNING_NOT_SEEN	2	/**< The application informs the library that the TCP handshake has not been seen */
+
+/**
+ * Optional information about flow management (per packet)
+ */
+struct ndpi_flow_input_info {
+  unsigned char in_pkt_dir;
+  unsigned char seen_flow_beginning;
+};
+
 /* ******************* ********************* ****************** */
 /* ************************************************************ */
 
@@ -1150,6 +1175,7 @@ struct ndpi_detection_module_struct {
 
   /* Current packet */
   struct ndpi_packet_struct packet;
+  const struct ndpi_flow_input_info *input_info;
 };
 
 #endif /* NDPI_LIB_COMPILATION */
@@ -1182,7 +1208,7 @@ struct ndpi_flow_struct {
   /* init parameter, internal used to set up timestamp,... */
   u_int16_t guessed_protocol_id, guessed_host_protocol_id, guessed_category, guessed_header_category;
   u_int8_t l4_proto, protocol_id_already_guessed:1, host_already_guessed:1, fail_with_unknown:1,
-    init_finished:1, setup_packet_direction:1, packet_direction:1, check_extra_packets:1, is_ipv6:1;
+    init_finished:1, client_packet_direction:1, packet_direction:1, check_extra_packets:1, is_ipv6:1;
   u_int16_t num_dissector_calls;
   ndpi_confidence_t confidence; /* ndpi_confidence_t */
 
@@ -1192,14 +1218,15 @@ struct ndpi_flow_struct {
   */
   u_int32_t next_tcp_seq_nr[2];
 
-  /* Flow addresses (used mainly for LRU lookups in ndpi_detection_giveup())
-     and ports. All in *network* byte order
-
-     TODO
-     - IPv6. Note that LRU is ipv4 only, for the time being 
+  /* Flow addresses (useful for LRU lookups in ndpi_detection_giveup())
+     and ports. All in *network* byte order.
+     Client and server.
    */
-  u_int32_t saddr, daddr;
-  u_int16_t sport, dport;
+  union {
+    u_int32_t v4;
+    u_int8_t v6[16];
+  } c_address, s_address;	/* For some unknown reasons, x86_64-w64-mingw32-gcc doesn't like the name "s_addr" */
+  u_int16_t c_port, s_port;
 
   // -----------------------------------------