Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Ookla: rework detection #1922

Merged
merged 1 commit into from
Mar 30, 2023
+258 −176
Merged

Ookla: rework detection #1922

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 43 additions & 2 deletions example/ndpiReader.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ u_int8_t verbose = 0, enable_flow_stats = 0;
int nDPI_LogLevel = 0;
char *_debug_protocols = NULL;
char *_disabled_protocols = NULL;
int aggressiveness[NDPI_MAX_SUPPORTED_PROTOCOLS];
static u_int8_t stats_flag = 0;
ndpi_init_prefs init_prefs = ndpi_no_prefs;
u_int8_t human_readeable_string_len = 5;
Expand Down Expand Up @@ -515,6 +516,7 @@ static void help(u_int long_help) {
" -z | Enable JA3+\n"
" -A | Dump internal statistics (LRU caches / Patricia trees / Ahocarasick automas / ...\n"
" -M | Memory allocation stats on data-path (only by the library). It works only on single-thread configuration\n"
" -Z proto:value | Set this value of aggressiveness for this protocol (0 to disable it). This flag can be used multiple times\n"
,
human_readeable_string_len,
min_pattern_len, max_pattern_len, max_num_packets_per_flow, max_packet_payload_dissection,
Expand Down Expand Up @@ -797,7 +799,7 @@ void printCSVHeader() {
*/
static void parseOptions(int argc, char **argv) {
int option_idx = 0;
int opt;
int opt, i;
#ifndef USE_DPDK
char *__pcap_file = NULL;
int thread_id, do_capture = 0;
Expand All @@ -818,7 +820,10 @@ static void parseOptions(int argc, char **argv) {
}
#endif

while((opt = getopt_long(argc, argv, "a:Ab:B:e:Ec:C:dDf:g:i:Ij:k:K:S:hHp:pP:l:r:s:tu:v:V:n:rp:x:w:zq0123:456:7:89:m:MT:U:",
for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++)
aggressiveness[i] = -1; /* Use the default value */

while((opt = getopt_long(argc, argv, "a:Ab:B:e:Ec:C:dDf:g:i:Ij:k:K:S:hHp:pP:l:r:s:tu:v:V:n:rp:x:w:zZ:q0123:456:7:89:m:MT:U:",
longopts, &option_idx)) != EOF) {
#ifdef DEBUG_TRACE
if(trace) fprintf(trace, " #### Handling option -%c [%s] #### \n", opt, optarg ? optarg : "");
Expand Down Expand Up @@ -950,6 +955,35 @@ static void parseOptions(int argc, char **argv) {
_disabled_protocols = ndpi_strdup(optarg);
break;

case 'Z': /* proto_name:aggr_value */
{
struct ndpi_detection_module_struct *module_tmp;
NDPI_PROTOCOL_BITMASK all;
char *saveptr, *tmp_str, *proto_str, *aggr_str;

/* Use a temporary module with all protocols enabled */
module_tmp = ndpi_init_detection_module(0);
if(!module_tmp)
break;
NDPI_BITMASK_SET_ALL(all);
ndpi_set_protocol_detection_bitmask2(module_tmp, &all);
ndpi_finalize_initialization(module_tmp);

tmp_str = ndpi_strdup(optarg);
if(tmp_str) {
proto_str = strtok_r(tmp_str, ":", &saveptr);
if(proto_str) {
aggr_str = strtok_r(NULL, ":", &saveptr);
if(aggr_str) {
aggressiveness[ndpi_get_protocol_id(module_tmp, proto_str)] = atoi(aggr_str);
}
}
}
ndpi_free(tmp_str);
ndpi_exit_detection_module(module_tmp);
break;
}

case 'h':
help(0);
break;
Expand Down Expand Up @@ -2413,6 +2447,7 @@ static void debug_printf(u_int32_t protocol, void *id_struct,
static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
NDPI_PROTOCOL_BITMASK enabled_bitmask;
struct ndpi_workflow_prefs prefs;
int i;

memset(&prefs, 0, sizeof(prefs));
prefs.decode_tunnels = decode_tunnels;
Expand Down Expand Up @@ -2472,6 +2507,12 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
NDPI_LRUCACHE_BITTORRENT, 32768);
/* Enable/disable LRU caches TTL here */

/* Set aggressiviness here */
IvanNardi marked this conversation as resolved.
Show resolved Hide resolved
for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++) {
if(aggressiveness[i] != -1)
ndpi_set_protocol_aggressiveness(ndpi_thread_info[thread_id].workflow->ndpi_struct, i, aggressiveness[i]);
}

ndpi_finalize_initialization(ndpi_thread_info[thread_id].workflow->ndpi_struct);

if(enable_doh_dot_detection)
Expand Down
5 changes: 5 additions & 0 deletions fuzz/fuzz_config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,11 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
ndpi_set_opportunistic_tls(ndpi_info_mod, random_proto, random_value);
ndpi_get_opportunistic_tls(ndpi_info_mod, random_proto);

for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++) {
ndpi_set_protocol_aggressiveness(ndpi_info_mod, i, random_value);
ndpi_get_protocol_aggressiveness(ndpi_info_mod, i);
}

ndpi_finalize_initialization(ndpi_info_mod);

/* Random protocol configuration */
Expand Down
5 changes: 5 additions & 0 deletions src/include/ndpi_api.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1031,6 +1031,11 @@ extern "C" {
int ndpi_get_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct,
u_int16_t proto);

int ndpi_set_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct,
u_int16_t proto, u_int32_t value);
u_int32_t ndpi_get_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct,
u_int16_t proto);

/**
* Find a protocol id associated with a string automata
*
Expand Down
17 changes: 16 additions & 1 deletion src/include/ndpi_typedefs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -659,6 +659,14 @@ struct ndpi_lru_cache {
struct ndpi_lru_cache_entry *entries;
};


/* Aggressiveness values */

#define NDPI_AGGRESSIVENESS_DISABLED 0x00 /* For all protocols */

/* Ookla */
#define NDPI_AGGRESSIVENESS_OOKLA_TLS 0x01 /* Enable detection over TLS (using ookla cache) */

/* ************************************************** */

struct ndpi_flow_tcp_struct {
Expand Down Expand Up @@ -920,7 +928,8 @@ typedef enum {
NDPI_CONFIDENCE_DPI_CACHE, /* Classification results based on some LRU cache (i.e. correlation among sessions) */
NDPI_CONFIDENCE_DPI, /* Deep packet inspection */
NDPI_CONFIDENCE_MATCH_BY_IP, /* Classification obtained looking only at the IP addresses */

NDPI_CONFIDENCE_DPI_AGGRESSIVE, /* Aggressive DPI: it might be a false positive */

/*
IMPORTANT
Expand Down Expand Up @@ -1229,6 +1238,8 @@ struct ndpi_detection_module_struct {
int opportunistic_tls_pop_enabled;
int opportunistic_tls_ftp_enabled;

u_int32_t aggressiveness_ookla;

u_int16_t ndpi_to_user_proto_id[NDPI_MAX_NUM_CUSTOM_PROTOCOLS]; /* custom protocolId mapping */
ndpi_proto_defaults_t proto_defaults[NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS];

Expand Down Expand Up @@ -1563,6 +1574,10 @@ struct ndpi_flow_struct {
/* NDPI_PROTOCOL_Z3950 */
u_int8_t z3950_stage : 2; // 0-3

/* NDPI_PROTOCOL_OOKLA */
u_int8_t ookla_stage : 1;


/* NDPI_PROTOCOL_OPENVPN */
u_int8_t ovpn_session_id[8];
u_int8_t ovpn_counter;
Expand Down
56 changes: 55 additions & 1 deletion src/lib/ndpi_main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,10 @@ extern void ndpi_unset_risk(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow, ndpi_risk_enum r);
extern u_int32_t make_mining_key(struct ndpi_flow_struct *flow);
extern int stun_search_into_zoom_cache(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow);
extern void ookla_add_to_cache(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow);
extern int ookla_search_into_cache(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow);

/* Forward */
static int addDefaultPort(struct ndpi_detection_module_struct *ndpi_str,
Expand Down Expand Up @@ -2932,7 +2936,7 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
ndpi_str->msteams_cache_num_entries = 1024;
ndpi_str->stun_zoom_cache_num_entries = 1024;

ndpi_str->ookla_cache_ttl = 0;
ndpi_str->ookla_cache_ttl = 120; /* sec */
ndpi_str->bittorrent_cache_ttl = 0;
ndpi_str->zoom_cache_ttl = 0;
ndpi_str->stun_cache_ttl = 0;
Expand All @@ -2946,6 +2950,8 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
ndpi_str->opportunistic_tls_pop_enabled = 1;
ndpi_str->opportunistic_tls_ftp_enabled = 1;

ndpi_str->aggressiveness_ookla = NDPI_AGGRESSIVENESS_OOKLA_TLS;

for(i = 0; i < NUM_CUSTOM_CATEGORIES; i++)
ndpi_snprintf(ndpi_str->custom_category_labels[i], CUSTOM_CATEGORY_LABEL_LEN, "User custom category %u",
(unsigned int) (i + 1));
Expand Down Expand Up @@ -6254,6 +6260,13 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
ret.app_protocol = flow->detected_protocol_stack[0];
}

/* Does it looks like Ookla? */
if(ret.app_protocol == NDPI_PROTOCOL_UNKNOWN &&
ntohs(flow->s_port) == 8080 && ookla_search_into_cache(ndpi_str, flow)) {
ndpi_set_detected_protocol(ndpi_str, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_PARTIAL_CACHE);
ret.app_protocol = flow->detected_protocol_stack[0];
}

/* Classification by-port is the last resort */
if(enable_guess && ret.app_protocol == NDPI_PROTOCOL_UNKNOWN) {

Expand Down Expand Up @@ -8052,6 +8065,9 @@ const char *ndpi_confidence_get_name(ndpi_confidence_t confidence)
case NDPI_CONFIDENCE_MATCH_BY_IP:
return "Match by IP";

case NDPI_CONFIDENCE_DPI_AGGRESSIVE:
return "DPI (aggressive)";

default:
return NULL;
}
Expand Down Expand Up @@ -8572,6 +8588,11 @@ int ndpi_match_hostname_protocol(struct ndpi_detection_module_struct *ndpi_struc
ndpi_set_detected_protocol(ndpi_struct, flow, subproto, master_protocol, NDPI_CONFIDENCE_DPI);
if(!category_depends_on_master(master_protocol))
ndpi_int_change_category(ndpi_struct, flow, ret_match.protocol_category);

if(subproto == NDPI_PROTOCOL_OOKLA) {
ookla_add_to_cache(ndpi_struct, flow);
}

return(1);
} else
return(0);
Expand Down Expand Up @@ -9643,3 +9664,36 @@ int ndpi_get_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct,
return -1;
}
}

/* ******************************************************************** */

int ndpi_set_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct,
u_int16_t proto, u_int32_t value)
{
if(!ndpi_struct)
return -1;

switch(proto) {
case NDPI_PROTOCOL_OOKLA:
ndpi_struct->aggressiveness_ookla = value;
return 0;
default:
return -1;
}
}

/* ******************************************************************** */

u_int32_t ndpi_get_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct,
u_int16_t proto)
{
if(!ndpi_struct)
return -1;

switch(proto) {
case NDPI_PROTOCOL_OOKLA:
return ndpi_struct->aggressiveness_ookla;
default:
return -1;
}
}
60 changes: 5 additions & 55 deletions src/lib/protocols/http.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ static const char* binary_file_ext[] = {
NULL
};

extern void ookla_add_to_cache(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow);

static void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow);

Expand Down Expand Up @@ -436,6 +439,7 @@ static void ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct *ndp
|| (strstr(flow->http.url, ":8080/upload?n=0.") != NULL))) {
/* This looks like Ookla speedtest */
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, master_protocol, NDPI_CONFIDENCE_DPI);
ookla_add_to_cache(ndpi_struct, flow);
}
}

Expand Down Expand Up @@ -1217,30 +1221,6 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
return;
}

if((packet->payload_packet_len == 3) && memcmp(packet->payload, "HI\n", 3) == 0) {
/* This looks like Ookla: we don't give up with HTTP yet */
flow->l4.tcp.http_stage = 1;
return;
}

if((packet->payload_packet_len == 40) && (flow->l4.tcp.http_stage == 0)) {
/*
-> QR O06L0072-6L91-4O43-857J-K8OO172L6L51
<- QNUUX 2.5 2017-08-15.1314.4jn12m5
-> MXFWUXJM 31625365
*/

if((packet->payload[2] == ' ')
&& (packet->payload[11] == '-')
&& (packet->payload[16] == '-')
&& (packet->payload[21] == '-')
&& (packet->payload[26] == '-')
&& (packet->payload[39] == 0x0A)
)
flow->l4.tcp.http_stage = 1;
return;
}

if((packet->payload_packet_len == 23) && (memcmp(packet->payload, "<policy-file-request/>", 23) == 0)) {
/*
<policy-file-request/>
Expand All @@ -1251,25 +1231,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
*/
ookla_found:
ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_CATEGORY_WEB);

if(ndpi_struct->ookla_cache != NULL) {
if(packet->iph != NULL) {
if(packet->tcp->source == htons(8080))
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->saddr, 1 /* dummy */, ndpi_get_current_time(flow));
else
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->daddr, 1 /* dummy */, ndpi_get_current_time(flow));
} else if(packet->iphv6 != NULL) {
u_int32_t h;

if(packet->tcp->source == htons(8080))
h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_src, sizeof(packet->iphv6->ip6_src));
else
h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_dst, sizeof(packet->iphv6->ip6_dst));

ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, h, 1 /* dummy */, ndpi_get_current_time(flow));
}
}

ookla_add_to_cache(ndpi_struct, flow);
return;
}

Expand Down Expand Up @@ -1389,18 +1351,6 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
} else if((flow->l4.tcp.http_stage == 1) || (flow->l4.tcp.http_stage == 2)) {
NDPI_LOG_DBG2(ndpi_struct, "HTTP stage %u: \n", flow->l4.tcp.http_stage);

if((packet->payload_packet_len == 34) && (flow->l4.tcp.http_stage == 1)) {
if((packet->payload[5] == ' ') && (packet->payload[9] == ' ')) {
goto ookla_found;
}
}

if((packet->payload_packet_len > 6) && memcmp(packet->payload, "HELLO ", 6) == 0) {
/* This looks like Ookla */
goto ookla_found;
} else
NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_OOKLA);

/**
At first check, if this is for sure a response packet
(in another direction. If not, if HTTP is detected do nothing now and return,
Expand Down
Loading