Add realtime protocol output to ndpiReader.

* support for using a new flow callback invoked before the flow memory is free'd * minor fixes * Win32 gmtime fix Signed-off-by: Toni Uhlig <[email protected]>
ntop · Dec 13, 2023 · bcc95cd · bcc95cd
1 parent ef62391
commit bcc95cd
Show file tree

Hide file tree

Showing 43 changed files with 338 additions and 20 deletions.
diff --git a/example/ndpiReader.c b/example/ndpiReader.c
@@ -2643,6 +2643,77 @@ static void debug_printf(u_int32_t protocol, void *id_struct,
 
 /* *********************************************** */
 
+static int is_realtime_protocol(ndpi_protocol proto)
+{
+  static u_int16_t const realtime_protos[] = {
+    NDPI_PROTOCOL_YOUTUBE,
+    NDPI_PROTOCOL_YOUTUBE_UPLOAD,
+    NDPI_PROTOCOL_TIKTOK,
+    NDPI_PROTOCOL_GOOGLE,
+    NDPI_PROTOCOL_GOOGLE_CLASSROOM,
+    NDPI_PROTOCOL_GOOGLE_CLOUD,
+    NDPI_PROTOCOL_GOOGLE_DOCS,
+    NDPI_PROTOCOL_GOOGLE_DRIVE,
+    NDPI_PROTOCOL_GOOGLE_MAPS,
+    NDPI_PROTOCOL_GOOGLE_SERVICES
+  };
+
+  for (u_int16_t i = 0; i < NDPI_ARRAY_LENGTH(realtime_protos); i++) {
+    if (proto.app_protocol == realtime_protos[i]
+        || proto.master_protocol == realtime_protos[i])
+    {
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+static void dump_realtime_protocol(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow)
+{
+  FILE *out = results_file ? results_file : stdout;
+  char srcip[64], dstip[64];
+  char ip_proto[64], app_name[64];
+  char date[64];
+  int ret = is_realtime_protocol(flow->detected_protocol);
+  time_t firsttime = flow->first_seen_ms;
+  struct tm result;
+
+  if (ndpi_gmtime_r(&firsttime, &result) != NULL)
+  {
+    strftime(date, sizeof(date), "%d.%m.%y %H:%M:%S", &result);
+  } else {
+    snprintf(date, sizeof(date), "%s", "Unknown");
+  }
+
+  if (flow->ip_version==4) {
+    inet_ntop(AF_INET, &flow->src_ip, srcip, sizeof(srcip));
+    inet_ntop(AF_INET, &flow->dst_ip, dstip, sizeof(dstip));
+  } else {
+    snprintf(srcip, sizeof(srcip), "[%s]", flow->src_name);
+    snprintf(dstip, sizeof(dstip), "[%s]", flow->dst_name);
+  }
+
+  ndpi_protocol2name(workflow->ndpi_struct, flow->detected_protocol, app_name, sizeof(app_name));
+
+  if (ret == 1) {
+    fprintf(out, "Detected Realtime protocol %s --> [%s] %s:%d <--> %s:%d app=%s <%s>\n",
+            date, ndpi_get_ip_proto_name(flow->protocol, ip_proto, sizeof(ip_proto)),
+            srcip, ntohs(flow->src_port), dstip, ntohs(flow->dst_port),
+            app_name, flow->human_readeable_string_buffer);
+  }
+}
+
+static void on_protocol_discovered(struct ndpi_workflow * workflow,
+                                   struct ndpi_flow_info * flow,
+                                   void * userdata)
+{
+  (void)userdata;
+  dump_realtime_protocol(workflow, flow);
+}
+
+/* *********************************************** */
+
 /**
  * @brief Setup for detection begin
  */
@@ -2701,6 +2772,9 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
     }
   }
 
+  ndpi_workflow_set_flow_callback(ndpi_thread_info[thread_id].workflow,
+                                  on_protocol_discovered, NULL);
+
   /* Make sure to load lists before finalizing the initialization */
   ndpi_set_protocol_detection_bitmask2(ndpi_thread_info[thread_id].workflow->ndpi_struct, &enabled_bitmask);
 

diff --git a/example/reader_util.c b/example/reader_util.c
@@ -1423,6 +1423,9 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
     flow->flow_payload = flow->ndpi_flow->flow_payload, flow->flow_payload_len = flow->ndpi_flow->flow_payload_len;
     flow->ndpi_flow->flow_payload = NULL; /* We'll free the memory */
 
+    if(workflow->flow_callback != NULL)
+      workflow->flow_callback(workflow, flow, workflow->flow_callback_userdata);
+
     ndpi_free_flow_info_half(flow);
   }
 }

diff --git a/example/reader_util.h b/example/reader_util.h
@@ -374,6 +374,9 @@ typedef struct ndpi_workflow {
   struct ndpi_workflow_prefs prefs;
   struct ndpi_stats stats;
 
+  ndpi_workflow_callback_ptr flow_callback;
+  void * flow_callback_userdata;
+
   /* outside referencies */
   pcap_t *pcap_handle;
 
@@ -408,6 +411,13 @@ struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
 					       const u_char *packet,
 					       ndpi_risk *flow_risk);
 
+
+/* Flow callback for completed flows, before the flow memory will be freed. */
+static inline void ndpi_workflow_set_flow_callback(struct ndpi_workflow * workflow, ndpi_workflow_callback_ptr callback, void * userdata) {
+  workflow->flow_callback = callback;
+  workflow->flow_callback_userdata = userdata;
+}
+
 int ndpi_is_datalink_supported(int datalink_type);
 
 /* compare two nodes in workflow */

diff --git a/src/include/ndpi_define.h.in b/src/include/ndpi_define.h.in
@@ -290,14 +290,15 @@
 			  ndpi_parse_packet_line_info(ndpi_struct,flow);	\
                         }                                                       \
 
-#define NDPI_IPSEC_PROTOCOL_ESP	   50
-#define NDPI_IPSEC_PROTOCOL_AH	   51
-#define NDPI_GRE_PROTOCOL_TYPE	   0x2F
-#define NDPI_ICMP_PROTOCOL_TYPE	   0x01
-#define NDPI_IGMP_PROTOCOL_TYPE	   0x02
-#define NDPI_EGP_PROTOCOL_TYPE	   0x08
-#define NDPI_OSPF_PROTOCOL_TYPE	   0x59
-#define NDPI_SCTP_PROTOCOL_TYPE	   132
+#define NDPI_IPSEC_PROTOCOL_ESP    50
+#define NDPI_IPSEC_PROTOCOL_AH     51
+#define NDPI_GRE_PROTOCOL_TYPE     0x2F
+#define NDPI_ICMP_PROTOCOL_TYPE    0x01
+#define NDPI_IGMP_PROTOCOL_TYPE    0x02
+#define NDPI_EGP_PROTOCOL_TYPE     0x08
+#define NDPI_OSPF_PROTOCOL_TYPE    0x59
+#define NDPI_VRRP_PROTOCOL_TYPE    112
+#define NDPI_SCTP_PROTOCOL_TYPE    132
 #define NDPI_IPIP_PROTOCOL_TYPE    0x04
 #define NDPI_ICMPV6_PROTOCOL_TYPE  0x3a
 #define NDPI_PGM_PROTOCOL_TYPE     0x71

diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
@@ -4081,7 +4081,7 @@ static u_int16_t guess_protocol_id(struct ndpi_detection_module_struct *ndpi_str
 	}
       }
       return(NDPI_PROTOCOL_IP_ICMPV6);
-    case 112:
+    case NDPI_VRRP_PROTOCOL_TYPE:
       return(NDPI_PROTOCOL_IP_VRRP);
     }
   }

diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c
@@ -1569,7 +1569,7 @@ char *ndpi_get_ip_proto_name(u_int16_t ip_proto, char *name, unsigned int name_l
     snprintf(name, name_len, "PIM");
     break;
 
-  case 112:
+  case NDPI_VRRP_PROTOCOL_TYPE:
     snprintf(name, name_len, "VRRP");
     break;
 
@@ -2815,8 +2815,8 @@ int ndpi_vsnprintf(char * str, size_t size, char const * format, va_list va_args
 struct tm *ndpi_gmtime_r(const time_t *timep,
                          struct tm *result)
 {
-#ifdef WIN32
-  gmtime_s(result, timep);
+#if defined(WIN32) && !defined(__GNUC__)
+  _gmtime64_s(result, timep);
   return result;
 #else
   return gmtime_r(timep, result);

diff --git a/tests/cfgs/default/result/1kxun.pcap.out b/tests/cfgs/default/result/1kxun.pcap.out
@@ -1,3 +1,8 @@
+Detected Realtime protocol 19.05.95 04:04:06 --> [TCP] 192.168.2.126:41390 <--> 18.64.79.37:80 app=HTTP.Google <SGET /doubleclick/ca0>
+Detected Realtime protocol 19.05.95 04:33:00 --> [TCP] 192.168.2.126:38354 <--> 142.250.186.34:80 app=HTTP.Google <GET /pagead/show>
+Detected Realtime protocol 19.05.95 04:42:17 --> [TCP] 192.168.2.126:36732 <--> 142.250.186.174:80 app=HTTP.Google <GET /analytics.js HTTP/1.1>
+Detected Realtime protocol 19.05.95 08:26:40 --> [TCP] 192.168.2.126:44368 <--> 172.217.18.98:80 app=HTTP.GoogleServices <GET /tag/js/gpt.j>
+Detected Realtime protocol 20.05.95 05:20:06 --> [TCP] 192.168.2.126:53416 <--> 172.217.16.142:80 app=HTTP.Google <GET /store/apps/details>
 Guessed flow protos:	6
 
 DPI Packets (TCP):	408	(4.16 pkts/flow)

diff --git a/tests/cfgs/default/result/EAQ.pcap.out b/tests/cfgs/default/result/EAQ.pcap.out
@@ -1,3 +1,5 @@
+Detected Realtime protocol 15.04.74 19:02:42 --> [TCP] 10.8.0.1:53497 <--> 173.194.119.48:80 app=HTTP.Google <GET / HTTP/1.1>
+Detected Realtime protocol 15.04.74 19:07:16 --> [TCP] 10.8.0.1:40467 <--> 173.194.119.24:80 app=HTTP.Google <we50oDAAg HTTP/1.1>
 DPI Packets (TCP):	12	(6.00 pkts/flow)
 DPI Packets (UDP):	116	(4.00 pkts/flow)
 Confidence DPI              : 31 (flows)

diff --git a/tests/cfgs/default/result/alexa-app.pcapng.out b/tests/cfgs/default/result/alexa-app.pcapng.out
@@ -1,3 +1,11 @@
+Detected Realtime protocol 23.02.17 22:53:13 --> [UDP] 172.16.42.216:3440 <--> 172.16.42.1:53 app=DNS.Google <connectivitycheck>
+Detected Realtime protocol 23.02.17 22:54:07 --> [UDP] 172.16.42.216:55619 <--> 172.16.42.1:53 app=DNS.Google <connectivitycheck>
+Detected Realtime protocol 23.02.17 22:54:17 --> [TCP] 172.16.42.216:60246 <--> 172.217.9.142:80 app=HTTP.Google <GET /generate>
+Detected Realtime protocol 23.02.17 23:38:42 --> [UDP] 172.16.42.216:52603 <--> 172.16.42.1:53 app=DNS.Google <google>
+Detected Realtime protocol 23.02.17 23:38:34 --> [UDP] 172.16.42.216:53188 <--> 172.16.42.1:53 app=DNS.GoogleServices <google>
+Detected Realtime protocol 23.02.17 23:39:27 --> [TCP] 172.16.42.216:42878 <--> 173.194.223.188:5228 app=TLS.GoogleServices <mtalk.google.com>
+Detected Realtime protocol 23.02.17 23:42:04 --> [UDP] 172.16.42.216:10462 <--> 172.16.42.1:53 app=DNS.Google <google>
+Detected Realtime protocol 23.02.17 23:42:13 --> [TCP] 172.16.42.216:35540 <--> 172.217.9.142:80 app=HTTP.Google <GET /generate>
 Guessed flow protos:	14
 
 DPI Packets (TCP):	850	(7.02 pkts/flow)

diff --git a/tests/cfgs/default/result/android.pcap.out b/tests/cfgs/default/result/android.pcap.out
@@ -1,3 +1,24 @@
+Detected Realtime protocol 31.12.15 20:33:57 --> [UDP] 192.168.2.16:34540 <--> 192.168.2.1:53 app=DNS.Google <clients>
+Detected Realtime protocol 31.12.15 20:35:23 --> [UDP] 192.168.2.16:54837 <--> 192.168.2.1:53 app=DNS.GoogleServices <googleapis>
+Detected Realtime protocol 31.12.15 20:34:48 --> [TCP] 192.168.2.16:32974 <--> 216.239.38.120:443 app=TLS.Google <clients>
+Detected Realtime protocol 31.12.15 20:45:48 --> [TCP] 192.168.2.16:52486 <--> 172.217.20.74:443 app=TLS.GoogleServices <play.googleapis.com>
+Detected Realtime protocol 31.12.15 20:47:42 --> [UDP] 192.168.2.16:47081 <--> 192.168.2.1:53 app=DNS.Google <connectivitycheck>
+Detected Realtime protocol 31.12.15 20:49:57 --> [UDP] 192.168.2.16:51430 <--> 192.168.2.1:53 app=DNS.Google <measurement>
+Detected Realtime protocol 31.12.15 20:48:47 --> [TCP] 192.168.2.16:36890 <--> 172.217.18.3:443 app=TLS.Google <connectivitycheck.gstatic.com>
+Detected Realtime protocol 31.12.15 20:48:31 --> [TCP] 192.168.2.16:36888 <--> 172.217.18.3:443 app=TLS.Google <connectivitycheck.gstatic.com>
+Detected Realtime protocol 31.12.15 21:02:41 --> [UDP] 192.168.2.16:39008 <--> 192.168.2.1:53 app=DNS.GoogleServices <google>
+Detected Realtime protocol 31.12.15 21:05:17 --> [TCP] 192.168.2.16:50384 <--> 172.217.168.206:443 app=TLS.Google <measurement.com>
+Detected Realtime protocol 31.12.15 21:31:55 --> [UDP] 192.168.2.16:40580 <--> 192.168.2.1:53 app=DNS.Google <google>
+Detected Realtime protocol 31.12.15 21:32:32 --> [TCP] 192.168.2.16:32996 <--> 216.239.38.120:443 app=TLS.Google <www.google.com>
+Detected Realtime protocol 31.12.15 21:34:52 --> [UDP] 192.168.2.16:46359 <--> 192.168.2.1:53 app=DNS.Google <accounts>
+Detected Realtime protocol 31.12.15 21:35:21 --> [TCP] 192.168.2.16:32998 <--> 216.239.38.120:443 app=TLS.Google <accounts.google.com>
+Detected Realtime protocol 31.12.15 21:35:43 --> [UDP] 192.168.2.16:35689 <--> 192.168.2.1:53 app=DNS.GoogleServices <semanticlocation>
+Detected Realtime protocol 31.12.15 21:40:00 --> [UDP] 192.168.2.16:58892 <--> 192.168.2.1:53 app=DNS.Google <accounts>
+Detected Realtime protocol 31.12.15 21:40:23 --> [TCP] 192.168.2.16:33002 <--> 216.239.38.120:443 app=TLS.Google <accounts.google.com>
+Detected Realtime protocol 31.12.15 21:43:47 --> [UDP] 192.168.2.16:32832 <--> 192.168.2.1:53 app=DNS.Google <google>
+Detected Realtime protocol 31.12.15 21:43:59 --> [TCP] 192.168.2.16:33014 <--> 216.239.38.120:443 app=TLS.Google <www.google.com>
+Detected Realtime protocol 31.12.15 21:44:41 --> [UDP] 192.168.2.16:39760 <--> 192.168.2.1:53 app=DNS.GoogleServices <android>
+Detected Realtime protocol 31.12.15 21:45:47 --> [TCP] 192.168.2.16:44374 <--> 172.217.22.10:443 app=TLS.GoogleServices <android.googleapis.com>
 Guessed flow protos:	3
 
 DPI Packets (TCP):	147	(5.25 pkts/flow)

diff --git a/tests/cfgs/default/result/dns_ambiguous_names.pcap.out b/tests/cfgs/default/result/dns_ambiguous_names.pcap.out
@@ -1,3 +1,5 @@
+Detected Realtime protocol 12.10.87 21:57:08 --> [UDP] 10.200.2.11:46134 <--> 8.8.8.8:53 app=DNS.GoogleServices <google>
+Detected Realtime protocol 12.10.87 21:59:37 --> [UDP] 10.200.2.11:44198 <--> 8.8.8.8:53 app=DNS.Google <youtube>
 DPI Packets (UDP):	20	(2.00 pkts/flow)
 Confidence DPI              : 10 (flows)
 Num dissector calls: 10 (1.00 diss/flow)

diff --git a/tests/cfgs/default/result/gquic.pcap.out b/tests/cfgs/default/result/gquic.pcap.out
@@ -1,3 +1,4 @@
+Detected Realtime protocol 19.07.14 21:32:58 --> [UDP] 10.44.5.25:61097 <--> 216.58.213.163:443 app=QUIC.Google <>
 DPI Packets (UDP):	1	(1.00 pkts/flow)
 Confidence DPI              : 1 (flows)
 Num dissector calls: 1 (1.00 diss/flow)

diff --git a/tests/cfgs/default/result/http_ipv6.pcap.out b/tests/cfgs/default/result/http_ipv6.pcap.out
@@ -1,3 +1,4 @@
+Detected Realtime protocol 28.10.63 03:50:00 --> [UDP] [2a00:d40:1:3:7aac:c0ff:fea7:d4c]:45931 <--> [2a00:1450:4001:803::1017]:443 app=QUIC.Google <www.google.it>
 Guessed flow protos:	7
 
 DPI Packets (TCP):	77	(5.92 pkts/flow)

diff --git a/tests/cfgs/default/result/ocs.pcap.out b/tests/cfgs/default/result/ocs.pcap.out
@@ -1,3 +1,4 @@
+Detected Realtime protocol 02.09.07 20:35:57 --> [TCP] 192.168.180.2:32946 <--> 64.233.184.188:443 app=TLS.GoogleServices <mtalk.google.com>
 Guessed flow protos:	2
 
 DPI Packets (TCP):	92	(7.67 pkts/flow)

diff --git a/tests/cfgs/default/result/pinterest.pcap.out b/tests/cfgs/default/result/pinterest.pcap.out
@@ -1,3 +1,9 @@
+Detected Realtime protocol 10.08.39 01:36:30 --> [TCP] [2a01:cb01:2049:8b07:991d:ec85:28df:f629]:40694 <--> [2a00:1450:4007:816::2004]:443 app=TLS.Google <www.google.com>
+Detected Realtime protocol 10.08.39 01:45:33 --> [TCP] [2a01:cb01:2049:8b07:991d:ec85:28df:f629]:51582 <--> [2a00:1450:4007:816::2003]:443 app=TLS.Google <www.gstatic.com>
+Detected Realtime protocol 10.08.39 01:47:01 --> [TCP] [2a01:cb01:2049:8b07:991d:ec85:28df:f629]:54416 <--> [2a00:1450:4007:806::200e]:443 app=TLS.Google <apis.google.com>
+Detected Realtime protocol 10.08.39 01:59:26 --> [TCP] [2a01:cb01:2049:8b07:991d:ec85:28df:f629]:47790 <--> [2a00:1450:4007:816::200a]:443 app=TLS.GoogleServices <content>
+Detected Realtime protocol 10.08.39 02:25:48 --> [TCP] [2a01:cb01:2049:8b07:991d:ec85:28df:f629]:40894 <--> [2a00:1450:4007:816::200d]:443 app=TLS.Google <accounts.google.com>
+Detected Realtime protocol 10.08.39 06:42:52 --> [TCP] [2a01:cb01:2049:8b07:991d:ec85:28df:f629]:45126 <--> [2a00:1450:4007:80a::200e]:443 app=TLS.Google <www.google>
 Guessed flow protos:	16
 
 DPI Packets (TCP):	216	(5.84 pkts/flow)

diff --git a/tests/cfgs/default/result/quic-27.pcap.out b/tests/cfgs/default/result/quic-27.pcap.out
@@ -1,3 +1,4 @@
+Detected Realtime protocol 08.10.30 13:05:15 --> [UDP] [3ef4:2194:f4a6:3503:40cd:714:57:c4e4]:64229 <--> [2f3d:64d1:9d59:549b::200e]:443 app=QUIC.Google <>
 DPI Packets (UDP):	1	(1.00 pkts/flow)
 Confidence DPI              : 1 (flows)
 Num dissector calls: 1 (1.00 diss/flow)

diff --git a/tests/cfgs/default/result/quic.pcap.out b/tests/cfgs/default/result/quic.pcap.out
@@ -1,3 +1,10 @@
+Detected Realtime protocol 23.07.32 03:51:41 --> [UDP] 192.168.1.105:45669 <--> 172.217.16.4:443 app=QUIC.Google <www.google.comO>
+Detected Realtime protocol 23.07.32 03:52:16 --> [UDP] 192.168.1.105:34438 <--> 216.58.210.238:443 app=QUIC.YouTube <www.youtube.com>
+Detected Realtime protocol 23.07.32 03:52:29 --> [UDP] 192.168.1.105:48445 <--> 216.58.214.110:443 app=QUIC.YouTube <i.ytimg.com>
+Detected Realtime protocol 23.07.32 03:52:36 --> [UDP] 192.168.1.105:40030 <--> 216.58.201.227:443 app=QUIC.Google <fonts.gstatic.com>
+Detected Realtime protocol 23.07.32 03:52:44 --> [UDP] 192.168.1.105:55934 <--> 216.58.201.238:443 app=QUIC.YouTube <s.ytimg.com>
+Detected Realtime protocol 23.07.32 03:52:58 --> [UDP] 192.168.1.105:53817 <--> 216.58.210.225:443 app=QUIC.YouTube <yt3.ggpht.com>
+Detected Realtime protocol 12.01.33 11:01:39 --> [UDP] 192.168.1.109:35236 <--> 216.58.210.206:443 app=QUIC.YouTube <www.youtube.com>
 Guessed flow protos:	1
 
 DPI Packets (UDP):	12	(1.20 pkts/flow)

diff --git a/tests/cfgs/default/result/quic046.pcap.out b/tests/cfgs/default/result/quic046.pcap.out
@@ -1,3 +1,4 @@
+Detected Realtime protocol 02.06.79 07:58:53 --> [UDP] 192.168.1.236:50587 <--> 216.58.206.86:443 app=QUIC.YouTube <i.ytimg.com>
 DPI Packets (UDP):	1	(1.00 pkts/flow)
 Confidence DPI              : 1 (flows)
 Num dissector calls: 1 (1.00 diss/flow)

diff --git a/tests/cfgs/default/result/quic_0RTT.pcap.out b/tests/cfgs/default/result/quic_0RTT.pcap.out
@@ -1,3 +1,4 @@
+Detected Realtime protocol 23.12.24 12:00:02 --> [UDP] 192.168.2.100:51972 <--> 142.250.181.227:443 app=QUIC.Google <>
 DPI Packets (UDP):	4	(2.00 pkts/flow)
 Confidence DPI              : 2 (flows)
 Num dissector calls: 199 (99.50 diss/flow)