diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 05f362ce909..0b7d5aa5eec 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -98,7 +98,7 @@ char *_debug_protocols = NULL; char *_disabled_protocols = NULL; int aggressiveness[NDPI_MAX_SUPPORTED_PROTOCOLS]; static u_int8_t stats_flag = 0; -ndpi_init_prefs init_prefs = ndpi_no_prefs; +ndpi_init_prefs init_prefs = ndpi_no_prefs | ndpi_enable_tcp_ack_payload_heuristic; u_int8_t human_readeable_string_len = 5; u_int8_t max_num_udp_dissected_pkts = 24 /* 8 is enough for most protocols, Signal and SnapchatCall require more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */; static u_int32_t pcap_analysis_duration = (u_int32_t)-1; diff --git a/fuzz/fuzz_ndpi_reader.c b/fuzz/fuzz_ndpi_reader.c index 252503d630b..42c72d573b1 100644 --- a/fuzz/fuzz_ndpi_reader.c +++ b/fuzz/fuzz_ndpi_reader.c @@ -18,7 +18,7 @@ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0; u_int8_t enable_flow_stats = 1; u_int8_t human_readeable_string_len = 5; u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */; -ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_ja3_plus; +ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_ja3_plus | ndpi_enable_tcp_ack_payload_heuristic; int enable_malloc_bins = 1; int malloc_size_stats = 0; int max_malloc_bins = 0; diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index d3ccd208cc4..2e8c76d2601 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -1273,6 +1273,8 @@ struct ndpi_detection_module_struct { u_int32_t aggressiveness_ookla; + int tcp_ack_paylod_heuristic; + u_int16_t ndpi_to_user_proto_id[NDPI_MAX_NUM_CUSTOM_PROTOCOLS]; /* custom protocolId mapping */ ndpi_proto_defaults_t proto_defaults[NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS]; @@ -1691,6 +1693,17 @@ typedef enum { ndpi_dont_init_risk_ptree = (1 << 14), ndpi_dont_load_cachefly_list = (1 << 15), ndpi_track_flow_payload = (1 << 16), + /* In some networks, there are some anomalous TCP flows where + the smallest ACK packets have some kind of zero padding. + It looks like the IP and TCP headers in those frames wrongly consider the + 0x00 Ethernet padding bytes as part of the TCP payload. + While this kind of packets is perfectly valid per-se, in some conditions + they might be treated by the TCP reassembler logic as (partial) overlaps, + deceiving the classification engine. + Add an heuristic to detect these packets and to ignore them, allowing + correct detection/classification. + See #1946 for other details */ + ndpi_enable_tcp_ack_payload_heuristic = (1 << 17), } ndpi_prefs; typedef struct { diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 763538e9633..9713ebe0c4b 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -2951,6 +2951,9 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs ndpi_str->aggressiveness_ookla = NDPI_AGGRESSIVENESS_OOKLA_TLS; + if(prefs & ndpi_enable_tcp_ack_payload_heuristic) + ndpi_str->tcp_ack_paylod_heuristic = 1; + for(i = 0; i < NUM_CUSTOM_CATEGORIES; i++) ndpi_snprintf(ndpi_str->custom_category_labels[i], CUSTOM_CATEGORY_LABEL_LEN, "User custom category %u", (unsigned int) (i + 1)); @@ -5492,6 +5495,20 @@ static u_int8_t ndpi_is_multi_or_broadcast(struct ndpi_packet_struct *packet) { /* ************************************************ */ +static int tcp_ack_padding(struct ndpi_packet_struct *packet) { + const struct ndpi_tcphdr *tcph = packet->tcp; + if(tcph && tcph->ack && !tcph->psh && + packet->payload_packet_len < 8 && + packet->payload_packet_len > 1 /* To avoid TCP keep-alives */) { + int i; + for(i = 0; i < packet->payload_packet_len; i++) + if(packet->payload[i] != 0) + return 0; + return 1; + } + return 0; +} + void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow) { if(!flow) { @@ -5580,7 +5597,10 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str, } } - if(flow->next_tcp_seq_nr[0] == 0 || flow->next_tcp_seq_nr[1] == 0 || + if(ndpi_str->tcp_ack_paylod_heuristic && tcp_ack_padding(packet)) { + NDPI_LOG_DBG2(ndpi_str, "TCP ACK with zero padding. Ignoring\n"); + packet->tcp_retransmission = 1; + } else if(flow->next_tcp_seq_nr[0] == 0 || flow->next_tcp_seq_nr[1] == 0 || (tcph->syn && flow->packet_counter == 0)) { /* initialize tcp sequence counters */ /* the ack flag needs to be set to get valid sequence numbers from the other diff --git a/tests/cfgs/default/pcap/heuristic_tcp_ack_payload.pcap b/tests/cfgs/default/pcap/heuristic_tcp_ack_payload.pcap new file mode 100644 index 00000000000..0aaf81ce534 Binary files /dev/null and b/tests/cfgs/default/pcap/heuristic_tcp_ack_payload.pcap differ diff --git a/tests/cfgs/default/result/heuristic_tcp_ack_payload.pcap.out b/tests/cfgs/default/result/heuristic_tcp_ack_payload.pcap.out new file mode 100644 index 00000000000..7e8fff675e5 --- /dev/null +++ b/tests/cfgs/default/result/heuristic_tcp_ack_payload.pcap.out @@ -0,0 +1,36 @@ +Guessed flow protos: 0 + +DPI Packets (TCP): 41 (10.25 pkts/flow) +Confidence DPI : 4 (flows) +Num dissector calls: 15 (3.75 diss/flow) +LRU cache ookla: 0/0/0 (insert/search/found) +LRU cache bittorrent: 0/0/0 (insert/search/found) +LRU cache zoom: 0/0/0 (insert/search/found) +LRU cache stun: 0/0/0 (insert/search/found) +LRU cache tls_cert: 0/6/0 (insert/search/found) +LRU cache mining: 0/0/0 (insert/search/found) +LRU cache msteams: 0/0/0 (insert/search/found) +LRU cache stun_zoom: 0/0/0 (insert/search/found) +Automa host: 8/1 (search/found) +Automa domain: 8/0 (search/found) +Automa tls cert: 2/0 (search/found) +Automa risk mask: 0/0 (search/found) +Automa common alpns: 6/6 (search/found) +Patricia risk mask: 8/0 (search/found) +Patricia risk: 8/0 (search/found) +Patricia protocols: 6/2 (search/found) + +TLS 160 63162 3 +WindowsUpdate 19 2638 1 + +JA3 Host Stats: + IP Address # JA3C + 1 194.226.199.103 1 + 2 194.226.199.21 1 + 3 194.226.199.61 1 + + + 1 TCP 194.226.199.61:27453 <-> 35.241.9.150:443 [proto: 91/TLS][IP: 284/GoogleCloud][Encrypted][Confidence: DPI][DPI packets: 8][cat: Web/5][36 pkts/3477 bytes <-> 42 pkts/37330 bytes][Goodput ratio: 44/94][171.42 sec][Hostname/SNI: firefox.settings.services.mozilla.com][(Advertised) ALPNs: h2;http/1.1][(Negotiated) ALPN: h2][bytes ratio: -0.830 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 4196/3653 58250/58245 14929/14067][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 97/889 375/2878 73/1070][TLSv1.2][JA3C: fb0aa01abe9d8e4037eb3473ca6e2dca][ServerNames: firefox.settings.services.mozilla.com,main-2-cdn.prod.kinto.prod.cloudops.mozgcp.net][JA3S: 9d9ce860f1b1cbef07b019450cb368d8][Issuer: C=US, O=Let's Encrypt, CN=R3][Subject: CN=main-2-cdn.prod.kinto.prod.cloudops.mozgcp.net][Certificate SHA-1: 30:0D:22:77:6E:DA:4E:99:3E:AF:8A:D0:5C:7D:97:51:8B:E6:22:11][Firefox][Validity: 2023-04-04 08:33:24 - 2023-07-03 08:33:23][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 49,16,2,0,3,3,1,0,1,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0,11] + 2 TCP 194.226.199.21:58155 <-> 52.18.127.189:443 [proto: 91/TLS][IP: 265/AmazonAWS][Encrypted][Confidence: DPI][DPI packets: 13][cat: Web/5][28 pkts/6789 bytes <-> 35 pkts/8995 bytes][Goodput ratio: 78/79][130.64 sec][Hostname/SNI: bitrix.info][(Advertised) ALPNs: h2;http/1.1][(Negotiated) ALPN: h2][TLS Supported Versions: GREASE;TLSv1.3;TLSv1.2][bytes ratio: -0.140 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 5498/4834 45102/45058 12717/11564][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 242/257 1547/2974 352/535][TLSv1.2][JA3C: ab205d804ecf934209c2a1bb94f817e0][JA3S: bfc90d56141386ee83b56cda231cccfc][Chrome][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 46,22,6,6,0,0,0,0,0,0,1,0,6,0,1,1,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,1,1] + 3 TCP 194.226.199.103:62580 <-> 217.69.139.59:443 [proto: 91/TLS][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 8][cat: Web/5][11 pkts/1346 bytes <-> 8 pkts/5225 bytes][Goodput ratio: 55/92][7.28 sec][Hostname/SNI: portal.mail.ru][(Advertised) ALPNs: h2;http/1.1][(Negotiated) ALPN: http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: -0.590 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 808/58 5455/213 1711/90][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 122/653 623/2897 162/957][TLSv1.2][JA3C: e669667efb41c36f714c309243f41ca7][ServerNames: *.mail.ru,mail.ru][JA3S: 2b33c1374db4ddf06942f92373c0b54b][Issuer: C=BE, O=GlobalSign nv-sa, CN=GlobalSign RSA OV SSL CA 2018][Subject: C=RU, ST=Moscow, L=Moscow, O=VK LLC, CN=*.mail.ru][Certificate SHA-1: 9F:A2:43:EA:AA:62:15:13:44:0D:15:75:17:47:4C:6B:E5:8E:10:1E][Firefox][Validity: 2022-10-20 09:52:31 - 2023-11-21 09:52:30][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 68,0,0,5,0,0,0,0,11,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,5] + 4 TCP 194.226.199.226:34101 <-> 8.247.226.126:80 [proto: 7.147/HTTP.WindowsUpdate][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 12][cat: SoftwareUpdate/19][7 pkts/896 bytes <-> 12 pkts/1742 bytes][Goodput ratio: 56/62][0.04 sec][Hostname/SNI: 3.tlu.dl.delivery.mp.microsoft.com][bytes ratio: -0.321 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 7/1 12/11 6/3][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 128/145 550/1076 172/281][URL: 3.tlu.dl.delivery.mp.microsoft.com/filestreamingservice/files/b4f27514-1618-47a0-bcd4-5fcb469edb63?P1=1681888058&P2=404&P3=2&P4=VJ2Qv%2bUXzBGOULZmyshxlc8XXx4pLl7hoFcLgf1iS33rDGfm0tCVrTPvZN8tn8yWBSrA0idwdtOBFLQMjZCUkw%3d%3d][StatusCode: 0][User-Agent: Microsoft-Delivery-Optimization/10.0][PLAIN TEXT (GET /filestreamingservice/files)][Plen Bins: 89,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]