diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index 02cb7364bcd..452c0f5ae15 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -96,6 +96,7 @@ u_int8_t verbose = 0, enable_flow_stats = 0;
int nDPI_LogLevel = 0;
char *_debug_protocols = NULL;
char *_disabled_protocols = NULL;
+int aggressiveness[NDPI_MAX_SUPPORTED_PROTOCOLS];
static u_int8_t stats_flag = 0;
ndpi_init_prefs init_prefs = ndpi_no_prefs;
u_int8_t human_readeable_string_len = 5;
@@ -515,6 +516,7 @@ static void help(u_int long_help) {
" -z | Enable JA3+\n"
" -A | Dump internal statistics (LRU caches / Patricia trees / Ahocarasick automas / ...\n"
" -M | Memory allocation stats on data-path (only by the library). It works only on single-thread configuration\n"
+ " -Z proto:value | Set this value of aggressiveness for this protocol (0 to disable it). This flag can be used multiple times\n"
,
human_readeable_string_len,
min_pattern_len, max_pattern_len, max_num_packets_per_flow, max_packet_payload_dissection,
@@ -797,7 +799,7 @@ void printCSVHeader() {
*/
static void parseOptions(int argc, char **argv) {
int option_idx = 0;
- int opt;
+ int opt, i;
#ifndef USE_DPDK
char *__pcap_file = NULL;
int thread_id, do_capture = 0;
@@ -818,7 +820,10 @@ static void parseOptions(int argc, char **argv) {
}
#endif
- while((opt = getopt_long(argc, argv, "a:Ab:B:e:Ec:C:dDf:g:i:Ij:k:K:S:hHp:pP:l:r:s:tu:v:V:n:rp:x:w:zq0123:456:7:89:m:MT:U:",
+ for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++)
+ aggressiveness[i] = -1; /* Use the default value */
+
+ while((opt = getopt_long(argc, argv, "a:Ab:B:e:Ec:C:dDf:g:i:Ij:k:K:S:hHp:pP:l:r:s:tu:v:V:n:rp:x:w:zZ:q0123:456:7:89:m:MT:U:",
longopts, &option_idx)) != EOF) {
#ifdef DEBUG_TRACE
if(trace) fprintf(trace, " #### Handling option -%c [%s] #### \n", opt, optarg ? optarg : "");
@@ -950,6 +955,35 @@ static void parseOptions(int argc, char **argv) {
_disabled_protocols = ndpi_strdup(optarg);
break;
+ case 'Z': /* proto_name:aggr_value */
+ {
+ struct ndpi_detection_module_struct *module_tmp;
+ NDPI_PROTOCOL_BITMASK all;
+ char *saveptr, *tmp_str, *proto_str, *aggr_str;
+
+ /* Use a temporary module with all protocols enabled */
+ module_tmp = ndpi_init_detection_module(0);
+ if(!module_tmp)
+ break;
+ NDPI_BITMASK_SET_ALL(all);
+ ndpi_set_protocol_detection_bitmask2(module_tmp, &all);
+ ndpi_finalize_initialization(module_tmp);
+
+ tmp_str = ndpi_strdup(optarg);
+ if(tmp_str) {
+ proto_str = strtok_r(tmp_str, ":", &saveptr);
+ if(proto_str) {
+ aggr_str = strtok_r(NULL, ":", &saveptr);
+ if(aggr_str) {
+ aggressiveness[ndpi_get_protocol_id(module_tmp, proto_str)] = atoi(aggr_str);
+ }
+ }
+ }
+ ndpi_free(tmp_str);
+ ndpi_exit_detection_module(module_tmp);
+ break;
+ }
+
case 'h':
help(0);
break;
@@ -2413,6 +2447,7 @@ static void debug_printf(u_int32_t protocol, void *id_struct,
static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
NDPI_PROTOCOL_BITMASK enabled_bitmask;
struct ndpi_workflow_prefs prefs;
+ int i;
memset(&prefs, 0, sizeof(prefs));
prefs.decode_tunnels = decode_tunnels;
@@ -2472,6 +2507,12 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
NDPI_LRUCACHE_BITTORRENT, 32768);
/* Enable/disable LRU caches TTL here */
+ /* Set aggressiviness here */
+ for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++) {
+ if(aggressiveness[i] != -1)
+ ndpi_set_protocol_aggressiveness(ndpi_thread_info[thread_id].workflow->ndpi_struct, i, aggressiveness[i]);
+ }
+
ndpi_finalize_initialization(ndpi_thread_info[thread_id].workflow->ndpi_struct);
if(enable_doh_dot_detection)
diff --git a/fuzz/fuzz_config.cpp b/fuzz/fuzz_config.cpp
index de1bdd3e85d..2e0a27ff4c0 100644
--- a/fuzz/fuzz_config.cpp
+++ b/fuzz/fuzz_config.cpp
@@ -108,6 +108,11 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
ndpi_set_opportunistic_tls(ndpi_info_mod, random_proto, random_value);
ndpi_get_opportunistic_tls(ndpi_info_mod, random_proto);
+ for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++) {
+ ndpi_set_protocol_aggressiveness(ndpi_info_mod, i, random_value);
+ ndpi_get_protocol_aggressiveness(ndpi_info_mod, i);
+ }
+
ndpi_finalize_initialization(ndpi_info_mod);
/* Random protocol configuration */
diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h
index d1875b5d818..d3b02a4d57a 100644
--- a/src/include/ndpi_api.h
+++ b/src/include/ndpi_api.h
@@ -1031,6 +1031,11 @@ extern "C" {
int ndpi_get_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct,
u_int16_t proto);
+ int ndpi_set_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct,
+ u_int16_t proto, u_int32_t value);
+ u_int32_t ndpi_get_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct,
+ u_int16_t proto);
+
/**
* Find a protocol id associated with a string automata
*
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 6d560ef36fd..a5b1175bc4f 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -659,6 +659,14 @@ struct ndpi_lru_cache {
struct ndpi_lru_cache_entry *entries;
};
+
+/* Aggressiveness values */
+
+#define NDPI_AGGRESSIVENESS_DISABLED 0x00 /* For all protocols */
+
+/* Ookla */
+#define NDPI_AGGRESSIVENESS_OOKLA_TLS 0x01 /* Enable detection over TLS (using ookla cache) */
+
/* ************************************************** */
struct ndpi_flow_tcp_struct {
@@ -920,7 +928,8 @@ typedef enum {
NDPI_CONFIDENCE_DPI_CACHE, /* Classification results based on some LRU cache (i.e. correlation among sessions) */
NDPI_CONFIDENCE_DPI, /* Deep packet inspection */
NDPI_CONFIDENCE_MATCH_BY_IP, /* Classification obtained looking only at the IP addresses */
-
+ NDPI_CONFIDENCE_DPI_AGGRESSIVE, /* Aggressive DPI: it might be a false positive */
+
/*
IMPORTANT
@@ -1229,6 +1238,8 @@ struct ndpi_detection_module_struct {
int opportunistic_tls_pop_enabled;
int opportunistic_tls_ftp_enabled;
+ u_int32_t aggressiveness_ookla;
+
u_int16_t ndpi_to_user_proto_id[NDPI_MAX_NUM_CUSTOM_PROTOCOLS]; /* custom protocolId mapping */
ndpi_proto_defaults_t proto_defaults[NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS];
@@ -1563,6 +1574,10 @@ struct ndpi_flow_struct {
/* NDPI_PROTOCOL_Z3950 */
u_int8_t z3950_stage : 2; // 0-3
+ /* NDPI_PROTOCOL_OOKLA */
+ u_int8_t ookla_stage : 1;
+
+
/* NDPI_PROTOCOL_OPENVPN */
u_int8_t ovpn_session_id[8];
u_int8_t ovpn_counter;
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index f4266d87d03..dc5834549bc 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -198,6 +198,10 @@ extern void ndpi_unset_risk(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow, ndpi_risk_enum r);
extern u_int32_t make_mining_key(struct ndpi_flow_struct *flow);
extern int stun_search_into_zoom_cache(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow);
+extern void ookla_add_to_cache(struct ndpi_detection_module_struct *ndpi_struct,
+ struct ndpi_flow_struct *flow);
+extern int ookla_search_into_cache(struct ndpi_detection_module_struct *ndpi_struct,
+ struct ndpi_flow_struct *flow);
/* Forward */
static int addDefaultPort(struct ndpi_detection_module_struct *ndpi_str,
@@ -2932,7 +2936,7 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
ndpi_str->msteams_cache_num_entries = 1024;
ndpi_str->stun_zoom_cache_num_entries = 1024;
- ndpi_str->ookla_cache_ttl = 0;
+ ndpi_str->ookla_cache_ttl = 120; /* sec */
ndpi_str->bittorrent_cache_ttl = 0;
ndpi_str->zoom_cache_ttl = 0;
ndpi_str->stun_cache_ttl = 0;
@@ -2946,6 +2950,8 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
ndpi_str->opportunistic_tls_pop_enabled = 1;
ndpi_str->opportunistic_tls_ftp_enabled = 1;
+ ndpi_str->aggressiveness_ookla = NDPI_AGGRESSIVENESS_OOKLA_TLS;
+
for(i = 0; i < NUM_CUSTOM_CATEGORIES; i++)
ndpi_snprintf(ndpi_str->custom_category_labels[i], CUSTOM_CATEGORY_LABEL_LEN, "User custom category %u",
(unsigned int) (i + 1));
@@ -6254,6 +6260,13 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
ret.app_protocol = flow->detected_protocol_stack[0];
}
+ /* Does it looks like Ookla? */
+ if(ret.app_protocol == NDPI_PROTOCOL_UNKNOWN &&
+ ntohs(flow->s_port) == 8080 && ookla_search_into_cache(ndpi_str, flow)) {
+ ndpi_set_detected_protocol(ndpi_str, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_PARTIAL_CACHE);
+ ret.app_protocol = flow->detected_protocol_stack[0];
+ }
+
/* Classification by-port is the last resort */
if(enable_guess && ret.app_protocol == NDPI_PROTOCOL_UNKNOWN) {
@@ -8052,6 +8065,9 @@ const char *ndpi_confidence_get_name(ndpi_confidence_t confidence)
case NDPI_CONFIDENCE_MATCH_BY_IP:
return "Match by IP";
+ case NDPI_CONFIDENCE_DPI_AGGRESSIVE:
+ return "DPI (aggressive)";
+
default:
return NULL;
}
@@ -8572,6 +8588,11 @@ int ndpi_match_hostname_protocol(struct ndpi_detection_module_struct *ndpi_struc
ndpi_set_detected_protocol(ndpi_struct, flow, subproto, master_protocol, NDPI_CONFIDENCE_DPI);
if(!category_depends_on_master(master_protocol))
ndpi_int_change_category(ndpi_struct, flow, ret_match.protocol_category);
+
+ if(subproto == NDPI_PROTOCOL_OOKLA) {
+ ookla_add_to_cache(ndpi_struct, flow);
+ }
+
return(1);
} else
return(0);
@@ -9643,3 +9664,36 @@ int ndpi_get_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct,
return -1;
}
}
+
+/* ******************************************************************** */
+
+int ndpi_set_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct,
+ u_int16_t proto, u_int32_t value)
+{
+ if(!ndpi_struct)
+ return -1;
+
+ switch(proto) {
+ case NDPI_PROTOCOL_OOKLA:
+ ndpi_struct->aggressiveness_ookla = value;
+ return 0;
+ default:
+ return -1;
+ }
+}
+
+/* ******************************************************************** */
+
+u_int32_t ndpi_get_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct,
+ u_int16_t proto)
+{
+ if(!ndpi_struct)
+ return -1;
+
+ switch(proto) {
+ case NDPI_PROTOCOL_OOKLA:
+ return ndpi_struct->aggressiveness_ookla;
+ default:
+ return -1;
+ }
+}
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index e0f56c4e804..4f139b8d318 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -46,6 +46,9 @@ static const char* binary_file_ext[] = {
NULL
};
+extern void ookla_add_to_cache(struct ndpi_detection_module_struct *ndpi_struct,
+ struct ndpi_flow_struct *flow);
+
static void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow);
@@ -436,6 +439,7 @@ static void ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct *ndp
|| (strstr(flow->http.url, ":8080/upload?n=0.") != NULL))) {
/* This looks like Ookla speedtest */
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, master_protocol, NDPI_CONFIDENCE_DPI);
+ ookla_add_to_cache(ndpi_struct, flow);
}
}
@@ -1217,30 +1221,6 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
return;
}
- if((packet->payload_packet_len == 3) && memcmp(packet->payload, "HI\n", 3) == 0) {
- /* This looks like Ookla: we don't give up with HTTP yet */
- flow->l4.tcp.http_stage = 1;
- return;
- }
-
- if((packet->payload_packet_len == 40) && (flow->l4.tcp.http_stage == 0)) {
- /*
- -> QR O06L0072-6L91-4O43-857J-K8OO172L6L51
- <- QNUUX 2.5 2017-08-15.1314.4jn12m5
- -> MXFWUXJM 31625365
- */
-
- if((packet->payload[2] == ' ')
- && (packet->payload[11] == '-')
- && (packet->payload[16] == '-')
- && (packet->payload[21] == '-')
- && (packet->payload[26] == '-')
- && (packet->payload[39] == 0x0A)
- )
- flow->l4.tcp.http_stage = 1;
- return;
- }
-
if((packet->payload_packet_len == 23) && (memcmp(packet->payload, "", 23) == 0)) {
/*
@@ -1251,25 +1231,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
*/
ookla_found:
ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_CATEGORY_WEB);
-
- if(ndpi_struct->ookla_cache != NULL) {
- if(packet->iph != NULL) {
- if(packet->tcp->source == htons(8080))
- ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->saddr, 1 /* dummy */, ndpi_get_current_time(flow));
- else
- ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->daddr, 1 /* dummy */, ndpi_get_current_time(flow));
- } else if(packet->iphv6 != NULL) {
- u_int32_t h;
-
- if(packet->tcp->source == htons(8080))
- h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_src, sizeof(packet->iphv6->ip6_src));
- else
- h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_dst, sizeof(packet->iphv6->ip6_dst));
-
- ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, h, 1 /* dummy */, ndpi_get_current_time(flow));
- }
- }
-
+ ookla_add_to_cache(ndpi_struct, flow);
return;
}
@@ -1389,18 +1351,6 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
} else if((flow->l4.tcp.http_stage == 1) || (flow->l4.tcp.http_stage == 2)) {
NDPI_LOG_DBG2(ndpi_struct, "HTTP stage %u: \n", flow->l4.tcp.http_stage);
- if((packet->payload_packet_len == 34) && (flow->l4.tcp.http_stage == 1)) {
- if((packet->payload[5] == ' ') && (packet->payload[9] == ' ')) {
- goto ookla_found;
- }
- }
-
- if((packet->payload_packet_len > 6) && memcmp(packet->payload, "HELLO ", 6) == 0) {
- /* This looks like Ookla */
- goto ookla_found;
- } else
- NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_OOKLA);
-
/**
At first check, if this is for sure a response packet
(in another direction. If not, if HTTP is detected do nothing now and return,
diff --git a/src/lib/protocols/ookla.c b/src/lib/protocols/ookla.c
index 0da42212cca..d7764e630fa 100644
--- a/src/lib/protocols/ookla.c
+++ b/src/lib/protocols/ookla.c
@@ -23,111 +23,92 @@
#include "ndpi_api.h"
+/* #define DEBUG_OOKLA_LRU */
+
const u_int16_t ookla_port = 8080;
/* ************************************************************* */
-static void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct ndpi_flow_struct* flow) {
- struct ndpi_packet_struct* packet = &ndpi_struct->packet;
- u_int32_t addr = 0;
- u_int16_t sport, dport;
-
- NDPI_LOG_DBG(ndpi_struct, "Ookla detection\n");
-
- if(packet->tcp)
- sport = ntohs(packet->tcp->source), dport = htons(packet->tcp->dest);
+static u_int32_t get_ookla_key(struct ndpi_flow_struct *flow)
+{
+ if(flow->is_ipv6)
+ return ndpi_quick_hash(flow->c_address.v6, 16);
else
- sport = ntohs(packet->udp->source), dport = htons(packet->udp->dest);
+ return ntohl(flow->c_address.v4);
+}
- if((sport != ookla_port) && (dport != ookla_port)) {
-#ifdef OOKLA_DEBUG
- printf("=>>>>>>>> [OOKLA IPv6] Skipping flow [%u -> %u]\n", sport, dport);
-#endif
- goto ookla_exclude;
- }
-
- if(packet->iphv6 != NULL) {
- if((dport == ookla_port) && (packet->payload_packet_len >= 3)) {
- u_int32_t h;
-
- if((packet->payload_packet_len == 3)
- && (packet->payload[0] == 0x48) /* HI\n */
- && (packet->payload[1] == 0x49)
- && (packet->payload[2] == 0x0A)) {
- ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
-
- if(ndpi_struct->ookla_cache != NULL) {
- /* In order to avoid creating an IPv6 LRU we hash the IPv6 address */
- h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_dst, sizeof(packet->iphv6->ip6_dst));
-
-#ifdef OOKLA_DEBUG
- printf("=>>>>>>>> [OOKLA IPv6] Adding %u\n", h);
-#endif
- ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, h, 1 /* dummy */, ndpi_get_current_time(flow));
- }
- return;
- } else {
- if(sport == ookla_port)
- h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_src, sizeof(packet->iphv6->ip6_src));
- else
- h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_dst, sizeof(packet->iphv6->ip6_dst));
-
- if(ndpi_struct->ookla_cache != NULL) {
- u_int16_t dummy;
-
-#ifdef OOKLA_DEBUG
- printf("=>>>>>>>> [OOKLA IPv6] Searching %u\n", h);
-#endif
-
- if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, h, &dummy, 0 /* Don't remove it as it can be used for other connections */,
- ndpi_get_current_time(flow))) {
- NDPI_LOG_INFO(ndpi_struct, "found ookla tcp connection\n");
- ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_CACHE);
-#ifdef OOKLA_DEBUG
- printf("=>>>>> Found %u\n", h);
-#endif
- return;
- } else {
-#ifdef OOKLA_DEBUG
- printf("=>>>>> NOT Found %u\n", h);
-#endif
- }
- }
- }
- } else {
+/* ************************************************************* */
- goto ookla_exclude;
- }
- } else {
- if(sport == ookla_port)
- addr = packet->iph->saddr;
- else
- addr = packet->iph->daddr;
-
-#ifdef OOKLA_DEBUG
- printf("=>>>>>>>> [OOKLA IPv4] Searching %u\n", addr);
-#endif
-
- if(ndpi_struct->ookla_cache != NULL) {
- u_int16_t dummy;
-
- if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, addr, &dummy, 0 /* Don't remove it as it can be used for other connections */,
- ndpi_get_current_time(flow))) {
- NDPI_LOG_INFO(ndpi_struct, "found ookla tcp connection\n");
- ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_CACHE);
-#ifdef OOKLA_DEBUG
- printf("=>>>>> Found %u\n", addr);
+int ookla_search_into_cache(struct ndpi_detection_module_struct *ndpi_struct,
+ struct ndpi_flow_struct *flow)
+{
+ u_int32_t key;
+ u_int16_t dummy;
+
+ if(ndpi_struct->ookla_cache) {
+ key = get_ookla_key(flow);
+#ifdef DEBUG_OOKLA_LRU
+ printf("[LRU OOKLA] Search %u\n", key);
#endif
- return;
- } else {
-#ifdef OOKLA_DEBUG
- printf("=>>>>> NOT Found %u\n", addr);
+
+ if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, key,
+ &dummy, 0 /* Don't remove it as it can be used for other connections */,
+ ndpi_get_current_time(flow))) {
+#ifdef DEBUG_OOKLA_LRU
+ printf("[LRU OOKLA] Found\n");
#endif
- }
+ return 1;
}
}
+ return 0;
+}
+
+/* ************************************************************* */
+
+void ookla_add_to_cache(struct ndpi_detection_module_struct *ndpi_struct,
+ struct ndpi_flow_struct *flow)
+{
+ u_int32_t key;
+
+ if(ndpi_struct->ookla_cache) {
+ key = get_ookla_key(flow);
+#ifdef DEBUG_OOKLA_LRU
+ printf("[LRU OOKLA] ADDING %u\n", key);
+#endif
+ ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, key, 1 /* dummy */,
+ ndpi_get_current_time(flow));
+ }
+
+}
+
+/* ************************************************************* */
+
+void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct ndpi_flow_struct* flow) {
+ struct ndpi_packet_struct *packet = &ndpi_struct->packet;
+
+ NDPI_LOG_DBG(ndpi_struct, "Ookla detection\n");
+
+ if(ntohs(flow->s_port) != ookla_port && ntohs(flow->c_port) != ookla_port) {
+ NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
+ return;
+ }
+
+ if(flow->packet_counter == 1 &&
+ packet->payload_packet_len >= NDPI_STATICSTRING_LEN("HI") &&
+ memcmp(packet->payload, "HI", NDPI_STATICSTRING_LEN("HI")) == 0) {
+ flow->ookla_stage = 1;
+ return;
+ }
+ if(flow->packet_counter == 2 &&
+ flow->ookla_stage == 1 &&
+ packet->payload_packet_len >= NDPI_STATICSTRING_LEN("HELLO") &&
+ memcmp(packet->payload, "HELLO", NDPI_STATICSTRING_LEN("HELLO")) == 0) {
+ NDPI_LOG_INFO(ndpi_struct, "found ookla (Hi + Hello)\n");
+ ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
+ ookla_add_to_cache(ndpi_struct, flow);
+ return;
+ }
- ookla_exclude:
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
}
diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c
index 8e3fbedb823..58b19d0d5f2 100644
--- a/src/lib/protocols/tls.c
+++ b/src/lib/protocols/tls.c
@@ -33,6 +33,8 @@ extern int processClientServerHello(struct ndpi_detection_module_struct *ndpi_st
extern int http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
const u_int8_t *ua_ptr, u_int16_t ua_ptr_len);
+extern int ookla_search_into_cache(struct ndpi_detection_module_struct* ndpi_struct,
+ struct ndpi_flow_struct* flow);
/* QUIC/GQUIC stuff */
extern int quic_len(const uint8_t *buf, uint64_t *value);
extern int quic_len_buffer_still_required(uint8_t value);
@@ -1153,8 +1155,27 @@ static int ndpi_search_tls_tcp(struct ndpi_detection_module_struct *ndpi_struct,
#ifdef DEBUG_TLS_BLOCKS
printf("*** [TLS Block] No more blocks\n");
#endif
- flow->extra_packets_func = NULL;
- return(0); /* That's all */
+ /* An ookla flow? */
+ if((ndpi_struct->aggressiveness_ookla & NDPI_AGGRESSIVENESS_OOKLA_TLS) && /* Feature enabled */
+ (!something_went_wrong &&
+ flow->tls_quic.certificate_processed == 1 &&
+ flow->protos.tls_quic.hello_processed == 1) && /* TLS handshake found without errors */
+ flow->detected_protocol_stack[0] == NDPI_PROTOCOL_TLS && /* No IMAPS/FTPS/... */
+ flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN && /* No sub-classification */
+ ntohs(flow->s_port) == 8080 && /* Ookla port */
+ ookla_search_into_cache(ndpi_struct, flow)) {
+ NDPI_LOG_INFO(ndpi_struct, "found ookla (cache over TLS)\n");
+ /* Even if a LRU cache is involved, NDPI_CONFIDENCE_DPI_AGGRESSIVE seems more
+ suited than NDPI_CONFIDENCE_DPI_CACHE */
+ ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_TLS, NDPI_CONFIDENCE_DPI_AGGRESSIVE);
+ /* TLS over port 8080 usually triggers that risk; clear it */
+ ndpi_unset_risk(ndpi_struct, flow, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT);
+ flow->extra_packets_func = NULL;
+ return(0); /* That's all */
+ } else {
+ flow->extra_packets_func = NULL;
+ return(0); /* That's all */
+ }
} else
return(1);
}
diff --git a/tests/pcap/ookla.pcap b/tests/pcap/ookla.pcap
index 80123e85605..ac57c6d1765 100644
Binary files a/tests/pcap/ookla.pcap and b/tests/pcap/ookla.pcap differ
diff --git a/tests/result/alexa-app.pcapng.out b/tests/result/alexa-app.pcapng.out
index bcfa7270539..c7dc0f77ac0 100644
--- a/tests/result/alexa-app.pcapng.out
+++ b/tests/result/alexa-app.pcapng.out
@@ -7,7 +7,7 @@ Confidence Match by port : 5 (flows)
Confidence DPI (partial) : 9 (flows)
Confidence DPI : 146 (flows)
Num dissector calls: 473 (2.96 diss/flow)
-LRU cache ookla: 0/0/0 (insert/search/found)
+LRU cache ookla: 0/5/0 (insert/search/found)
LRU cache bittorrent: 0/42/0 (insert/search/found)
LRU cache zoom: 0/0/0 (insert/search/found)
LRU cache stun: 0/0/0 (insert/search/found)
diff --git a/tests/result/ookla.pcap.out b/tests/result/ookla.pcap.out
index 5923011c1e4..98af0162597 100644
--- a/tests/result/ookla.pcap.out
+++ b/tests/result/ookla.pcap.out
@@ -1,27 +1,37 @@
-Guessed flow protos: 0
+Guessed flow protos: 1
-DPI Packets (TCP): 16 (8.00 pkts/flow)
-Confidence DPI (cache) : 1 (flows)
-Confidence DPI : 1 (flows)
-Num dissector calls: 97 (48.50 diss/flow)
-LRU cache ookla: 3/1/1 (insert/search/found)
-LRU cache bittorrent: 0/0/0 (insert/search/found)
+DPI Packets (TCP): 46 (7.67 pkts/flow)
+Confidence DPI (partial cache): 1 (flows)
+Confidence DPI : 4 (flows)
+Confidence DPI (aggressive) : 1 (flows)
+Num dissector calls: 488 (81.33 diss/flow)
+LRU cache ookla: 6/2/2 (insert/search/found)
+LRU cache bittorrent: 0/3/0 (insert/search/found)
LRU cache zoom: 0/0/0 (insert/search/found)
LRU cache stun: 0/0/0 (insert/search/found)
-LRU cache tls_cert: 0/0/0 (insert/search/found)
-LRU cache mining: 0/0/0 (insert/search/found)
+LRU cache tls_cert: 0/2/0 (insert/search/found)
+LRU cache mining: 0/1/0 (insert/search/found)
LRU cache msteams: 0/0/0 (insert/search/found)
LRU cache stun_zoom: 0/0/0 (insert/search/found)
-Automa host: 0/0 (search/found)
-Automa domain: 0/0 (search/found)
+Automa host: 2/1 (search/found)
+Automa domain: 2/0 (search/found)
Automa tls cert: 0/0 (search/found)
-Automa risk mask: 0/0 (search/found)
-Automa common alpns: 0/0 (search/found)
-Patricia risk mask: 4/0 (search/found)
+Automa risk mask: 1/0 (search/found)
+Automa common alpns: 4/4 (search/found)
+Patricia risk mask: 12/0 (search/found)
Patricia risk: 0/0 (search/found)
-Patricia protocols: 4/0 (search/found)
+Patricia protocols: 11/1 (search/found)
-Ookla 50 6661 2
+Ookla 113 38411 6
- 1 TCP 192.168.1.7:51207 <-> 46.44.253.187:80 [proto: 7.191/HTTP.Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 12][cat: Network/14][12 pkts/2238 bytes <-> 8 pkts/2082 bytes][Goodput ratio: 64/74][5.33 sec][bytes ratio: 0.036 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/4 528/47 5005/84 1493/28][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 186/260 430/523 168/194][Risk: ** HTTP Susp User-Agent **** HTTP Obsolete Server **][Risk Score: 150][Risk Info: Obsolete Apache server 2.2.22 / Empty or missing User-Agent][PLAIN TEXT (GET /crossdomain.xml HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,12,75,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
- 2 TCP 192.168.1.7:51215 <-> 46.44.253.187:8080 [proto: 191/Ookla][IP: 0/Unknown][ClearText][Confidence: DPI (cache)][DPI packets: 4][cat: Network/14][19 pkts/1421 bytes <-> 11 pkts/920 bytes][Goodput ratio: 11/20][0.80 sec][bytes ratio: 0.214 (Upload)][IAT c2s/s2c min/avg/max/stddev: 26/0 44/75 103/137 23/41][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 75/84 85/100 9/8][PLAIN TEXT ( 6HELLO 2.4 2016)][Plen Bins: 94,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+JA3 Host Stats:
+ IP Address # JA3C
+ 1 192.168.1.128 2
+
+
+ 1 TCP 192.168.1.128:35830 <-> 89.96.108.170:8080 [proto: 91.191/TLS.Ookla][IP: 0/Unknown][Encrypted][Confidence: DPI (aggressive)][DPI packets: 6][cat: Web/5][21 pkts/21216 bytes <-> 8 pkts/1950 bytes][Goodput ratio: 93/72][0.32 sec][Hostname/SNI: spd-pub-mi-01-01.fastwebnet.it][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: 0.832 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 17/61 274/280 62/109][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 1010/244 1514/387 612/138][TLSv1.3][JA3C: c279b0189edb9269da7bc43dea5e0c36][JA3S: fcb2d4d0991292272fcb1e464eedfd43][Firefox][Cipher: TLS_AES_128_GCM_SHA256][Plen Bins: 0,0,4,0,0,0,0,4,9,0,9,0,0,0,0,0,4,0,0,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,55,0,0]
+ 2 TCP 192.168.1.128:48854 <-> 104.16.209.12:443 [proto: 91.191/TLS.Ookla][IP: 220/Cloudflare][Encrypted][Confidence: DPI][DPI packets: 6][cat: Network/14][8 pkts/1620 bytes <-> 6 pkts/3818 bytes][Goodput ratio: 67/89][0.06 sec][Hostname/SNI: www.speedtest.net][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: -0.404 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 7/5 18/15 7/6][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 202/636 583/1514 181/646][TLSv1.3][JA3C: 579ccef312d18482fc42e2b822ca2430][JA3S: eb1d94daa7e0344597e756a1fb6e7054][Firefox][Cipher: TLS_AES_128_GCM_SHA256][PLAIN TEXT (@oTAgOeedtest.net)][Plen Bins: 0,0,14,0,0,14,0,0,0,0,14,0,0,0,0,0,28,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,28,0,0]
+ 3 TCP 192.168.1.7:51207 <-> 46.44.253.187:80 [proto: 7.191/HTTP.Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 12][cat: Network/14][12 pkts/2238 bytes <-> 8 pkts/2082 bytes][Goodput ratio: 64/74][5.33 sec][bytes ratio: 0.036 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/4 528/47 5005/84 1493/28][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 186/260 430/523 168/194][Risk: ** HTTP Susp User-Agent **** HTTP Obsolete Server **][Risk Score: 150][Risk Info: Obsolete Apache server 2.2.22 / Empty or missing User-Agent][PLAIN TEXT (GET /crossdomain.xml HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,12,75,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+ 4 TCP 192.168.1.192:51156 <-> 89.96.108.170:8080 [proto: 191/Ookla][IP: 0/Unknown][ClearText][Confidence: DPI (partial cache)][DPI packets: 10][cat: Network/14][6 pkts/591 bytes <-> 4 pkts/1784 bytes][Goodput ratio: 32/85][0.05 sec][bytes ratio: -0.502 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 9/10 15/20 6/8][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 98/446 143/1514 31/617][PLAIN TEXT (gKRZvA)][Plen Bins: 0,40,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,0,0]
+ 5 TCP 192.168.1.7:51215 <-> 46.44.253.187:8080 [proto: 191/Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 6][cat: Network/14][19 pkts/1421 bytes <-> 11 pkts/920 bytes][Goodput ratio: 11/20][0.80 sec][bytes ratio: 0.214 (Upload)][IAT c2s/s2c min/avg/max/stddev: 26/0 44/75 103/137 23/41][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 75/84 85/100 9/8][PLAIN TEXT ( 6HELLO 2.4 2016)][Plen Bins: 94,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+ 6 TCP 192.168.1.192:37790 <-> 185.157.229.246:8080 [proto: 191/Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 6][cat: Network/14][6 pkts/454 bytes <-> 4 pkts/317 bytes][Goodput ratio: 11/14][0.06 sec][bytes ratio: 0.178 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 12/5 46/9 17/4][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 76/79 106/108 14/17][PLAIN TEXT (HELLO 2.9 )][Plen Bins: 50,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
diff --git a/tests/result/skype.pcap.out b/tests/result/skype.pcap.out
index 5091e3a4e5f..d26ba706f76 100644
--- a/tests/result/skype.pcap.out
+++ b/tests/result/skype.pcap.out
@@ -7,7 +7,7 @@ Confidence Unknown : 59 (flows)
Confidence Match by port : 27 (flows)
Confidence DPI (partial) : 1 (flows)
Confidence DPI : 206 (flows)
-Num dissector calls: 26899 (91.81 diss/flow)
+Num dissector calls: 26897 (91.80 diss/flow)
LRU cache ookla: 0/0/0 (insert/search/found)
LRU cache bittorrent: 0/261/0 (insert/search/found)
LRU cache zoom: 0/0/0 (insert/search/found)
diff --git a/tests/result/skype_no_unknown.pcap.out b/tests/result/skype_no_unknown.pcap.out
index be90096f61e..328c50280ed 100644
--- a/tests/result/skype_no_unknown.pcap.out
+++ b/tests/result/skype_no_unknown.pcap.out
@@ -6,7 +6,7 @@ DPI Packets (other): 5 (1.00 pkts/flow)
Confidence Unknown : 44 (flows)
Confidence Match by port : 22 (flows)
Confidence DPI : 201 (flows)
-Num dissector calls: 22405 (83.91 diss/flow)
+Num dissector calls: 22404 (83.91 diss/flow)
LRU cache ookla: 0/0/0 (insert/search/found)
LRU cache bittorrent: 0/198/0 (insert/search/found)
LRU cache zoom: 0/0/0 (insert/search/found)
diff --git a/tests/result/synscan.pcap.out b/tests/result/synscan.pcap.out
index 9932b97b2e1..3bdf713b463 100644
--- a/tests/result/synscan.pcap.out
+++ b/tests/result/synscan.pcap.out
@@ -4,7 +4,7 @@ DPI Packets (TCP): 2011 (1.01 pkts/flow)
Confidence Unknown : 1862 (flows)
Confidence Match by port : 132 (flows)
Num dissector calls: 0 (0.00 diss/flow)
-LRU cache ookla: 0/0/0 (insert/search/found)
+LRU cache ookla: 0/2/0 (insert/search/found)
LRU cache bittorrent: 0/5976/0 (insert/search/found)
LRU cache zoom: 0/0/0 (insert/search/found)
LRU cache stun: 0/0/0 (insert/search/found)