Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

FPC LRU cache related changes #2497

Merged
merged 10 commits into from
Jul 22, 2024
Merged

FPC LRU cache related changes #2497

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
d28b071
Add files via upload
mmanoj Jul 12, 2024
26d577f
Add files via upload
mmanoj Jul 12, 2024
36bea1e
Update for review changes
mmanoj Jul 14, 2024
0db15c6
Update as per review comments
mmanoj Jul 14, 2024
069d146
Merge branch 'ntop:dev' into dev
mmanoj Jul 20, 2024
fadd04b
FPC review change updates with DNS cache
mmanoj Jul 20, 2024
a2b012c
FPC review change updates with DNS cache
mmanoj Jul 20, 2024
c49c3d4
FPC review change updates with DNS cache
mmanoj Jul 20, 2024
11042e6
FPC review change updates with DNS cache
mmanoj Jul 20, 2024
2c12037
FPC (DNS): fix and improvements
IvanNardi Jul 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion src/include/ndpi_private.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,9 @@ struct ndpi_global_context {

/* NDPI_PROTOCOL_MSTEAMS */
struct ndpi_lru_cache *msteams_global_cache;

/* NDPI_FIRST_PKT_CLASSIFICATION_CACHE */
IvanNardi marked this conversation as resolved.
Show resolved Hide resolved
struct ndpi_lru_cache *fpc_dns_global_cache;
};

#define CFG_MAX_LEN 256
Expand Down Expand Up @@ -220,7 +223,10 @@ struct ndpi_detection_module_config_struct {
int msteams_cache_num_entries;
int msteams_cache_ttl;
int msteams_cache_scope;

int fpc_dns_cache_num_entries;
int fpc_dns_cache_ttl;
int fpc_dns_cache_scope;

/* Protocols */

int tls_certificate_expire_in_x_days;
Expand Down Expand Up @@ -361,6 +367,9 @@ struct ndpi_detection_module_struct {

/* NDPI_PROTOCOL_MSTEAMS */
struct ndpi_lru_cache *msteams_cache;

/* NDPI_FIRST_PKT_CLASSIFICATION_CACHE */
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As above

struct ndpi_lru_cache *fpc_dns_cache;

/* *** If you add a new LRU cache, please update lru_cache_type above! *** */

Expand Down
1 change: 1 addition & 0 deletions src/include/ndpi_typedefs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -747,6 +747,7 @@ typedef enum {
NDPI_LRUCACHE_TLS_CERT,
NDPI_LRUCACHE_MINING,
NDPI_LRUCACHE_MSTEAMS,
NDPI_LRUCACHE_FPC_DNS, /* First Pkt Classification */
IvanNardi marked this conversation as resolved.
Show resolved Hide resolved

NDPI_LRUCACHE_MAX /* Last one! */
} lru_cache_type;
Expand Down
91 changes: 89 additions & 2 deletions src/lib/ndpi_main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3260,6 +3260,8 @@ void ndpi_global_deinit(struct ndpi_global_context *g_ctx) {
ndpi_lru_free_cache(g_ctx->mining_global_cache);
if(g_ctx->msteams_global_cache)
ndpi_lru_free_cache(g_ctx->msteams_global_cache);
if(g_ctx->fpc_dns_global_cache)
ndpi_lru_free_cache(g_ctx->fpc_dns_global_cache);

ndpi_free(g_ctx);
}
Expand Down Expand Up @@ -3829,6 +3831,23 @@ int ndpi_finalize_initialization(struct ndpi_detection_module_struct *ndpi_str)
ndpi_str->cfg.msteams_cache_num_entries);
}
}
//ToDo:add fpc cache details
if(ndpi_str->cfg.fpc_dns_cache_num_entries > 0) {
if(ndpi_str->cfg.fpc_dns_cache_scope == NDPI_LRUCACHE_SCOPE_GLOBAL) {
if(!ndpi_str->g_ctx->fpc_dns_global_cache) {
ndpi_str->g_ctx->fpc_dns_global_cache = ndpi_lru_cache_init(ndpi_str->cfg.fpc_dns_cache_num_entries, ndpi_str->cfg.fpc_dns_cache_ttl, 1);
IvanNardi marked this conversation as resolved.
Show resolved Hide resolved
}
ndpi_str->fpc_dns_cache = ndpi_str->g_ctx->fpc_dns_global_cache;
} else {
ndpi_str->fpc_dns_cache = ndpi_lru_cache_init(ndpi_str->cfg.fpc_dns_cache_num_entries,
ndpi_str->cfg.fpc_dns_cache_ttl, 0);
}
if(!ndpi_str->fpc_dns_cache) {
NDPI_LOG_ERR(ndpi_str, "Error allocating lru fpc_dns_cache (num_entries %u)\n",
ndpi_str->cfg.fpc_dns_cache_num_entries);

}
}

ndpi_automa * const automa[] = { &ndpi_str->host_automa,
&ndpi_str->tls_cert_subject_automa,
Expand Down Expand Up @@ -4166,6 +4185,10 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
if(!ndpi_str->cfg.msteams_cache_scope &&
ndpi_str->msteams_cache)
ndpi_lru_free_cache(ndpi_str->msteams_cache);
//ToDo:add fpc details
IvanNardi marked this conversation as resolved.
Show resolved Hide resolved
if(!ndpi_str->cfg.fpc_dns_cache_scope &&
ndpi_str->fpc_dns_cache)
ndpi_lru_free_cache(ndpi_str->fpc_dns_cache);

if(ndpi_str->protocols) ndpi_ptree_destroy(ndpi_str->protocols);
if(ndpi_str->ip_risk_mask) ndpi_ptree_destroy(ndpi_str->ip_risk_mask);
Expand Down Expand Up @@ -7278,6 +7301,20 @@ u_int16_t ndpi_guess_host_protocol_id(struct ndpi_detection_module_struct *ndpi_

/* ********************************************************************************* */

u_int64_t make_fpc_dns_cache_key(struct ndpi_flow_struct *flow) {
u_int64_t key;

/* network byte order */
if(flow->is_ipv6)
key = (ndpi_quick_hash64((const char *)flow->c_address.v6, 16) << 32) | (ndpi_quick_hash64((const char *)flow->s_address.v6, 16) & 0xFFFFFFFF);
else
key = ((u_int64_t)flow->c_address.v4 << 32) | flow->s_address.v4;

return key;
}

/* ********************************************************************************* */

static u_int64_t make_msteams_key(struct ndpi_flow_struct *flow, u_int8_t use_client) {
u_int64_t key;

Expand Down Expand Up @@ -7369,6 +7406,21 @@ static void ndpi_reconcile_msteams_call_udp(struct ndpi_flow_struct *flow) {

/* ********************************************************************************* */

static void ndpi_fpc_cache_update(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow) {


if(ndpi_str->fpc_dns_cache){
// printf("\nInside ndpi_fpc_dns_cache_update()\n");
ndpi_lru_add_to_cache(ndpi_str->fpc_dns_cache,/*fpc_cache,*/
make_fpc_dns_cache_key(flow),
flow->fpc.app_protocol,
ndpi_get_current_time(flow));
} else{
}

}

static void ndpi_reconcile_protocols(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow,
ndpi_protocol *ret) {
Expand Down Expand Up @@ -8322,6 +8374,7 @@ static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detectio
NDPI_SELECTION_BITMASK_PROTOCOL_SIZE ndpi_selection_packet;
u_int32_t num_calls = 0;
ndpi_protocol ret;
u_int16_t fpc_dns_cached_proto;

memset(&ret, 0, sizeof(ret));

Expand Down Expand Up @@ -8375,6 +8428,31 @@ static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detectio

if(flow->num_processed_pkts == 1) {
/* first packet of this flow to be analyzed */

/*ToDo: First check in fpc cache, if found update the ndpi_flow_struct via fpc_update()
else update via below method from flow->guessed_protocol_id_by_ip, NDPI_FPC_CONFIDENCE_IP) and return(ret)
**use --> ndpi_lru_find_cache() metod and value return from it(dummy) as protocol
update the flow->fpc_app structue as results

*/

/* For each flow (at the very first packet) lookup into the cache using "SRC IP - DST IP" as key and, if a match is found, save the protocol id into ndpi_flow_struct structure as a "fpc result" */
//printf("\nApp proto from ret-->%u\n",ret.app_protocol);

/* Check some fpc cache */
if(ret.app_protocol == NDPI_PROTOCOL_UNKNOWN &&
ndpi_str->fpc_dns_cache &&
ndpi_lru_find_cache(ndpi_str->fpc_dns_cache, make_fpc_dns_cache_key(flow),
&fpc_dns_cached_proto, 0 /* Don't remove it as it can be used for other connections */,
ndpi_get_current_time(flow))) {
NDPI_LOG_DBG(ndpi_str,"\nFound from FPC cache...%u\n",fpc_dns_cached_proto);
ndpi_set_detected_protocol(ndpi_str, flow, fpc_dns_cached_proto, NDPI_PROTOCOL_UNKNOWN, NDPI_FPC_CONFIDENCE_DNS);
IvanNardi marked this conversation as resolved.
Show resolved Hide resolved
ret.app_protocol = fpc_dns_cached_proto;//flow->detected_protocol_stack[0];
/* Save the protocol id into ndpi_flow_struct structure as a "fpc result" */
fpc_update(ndpi_str, flow, NDPI_PROTOCOL_UNKNOWN,
fpc_dns_cached_proto, NDPI_FPC_CONFIDENCE_DNS);//testing
return(ret);
}

#ifdef HAVE_NBPF
if(ndpi_str->nbpf_custom_proto[0].tree != NULL) {
Expand Down Expand Up @@ -8448,6 +8526,8 @@ static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detectio
return(ret);

fpc_check_ip(ndpi_str, flow);
/* Update fpc cache */
ndpi_fpc_cache_update(ndpi_str,flow);
IvanNardi marked this conversation as resolved.
Show resolved Hide resolved
}

num_calls = ndpi_check_flow_func(ndpi_str, flow, &ndpi_selection_packet);
Expand Down Expand Up @@ -10250,6 +10330,10 @@ int ndpi_get_lru_cache_stats(struct ndpi_global_context *g_ctx,
case NDPI_LRUCACHE_MSTEAMS:
ndpi_lru_get_stats(is_local ? ndpi_struct->msteams_cache : g_ctx->msteams_global_cache, stats);
return 0;
//ToDo:add fpc cache details
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove the comment, please

case NDPI_LRUCACHE_FPC_DNS:
ndpi_lru_get_stats(is_local ? ndpi_struct->fpc_dns_cache : g_ctx->fpc_dns_global_cache, stats);
return 0;
default:
return -1;
}
Expand Down Expand Up @@ -11293,8 +11377,11 @@ static const struct cfg_param {
{ NULL, "lru.msteams.size", "1024", "0", "16777215", CFG_PARAM_INT, __OFF(msteams_cache_num_entries), NULL },
{ NULL, "lru.msteams.ttl", "60", "0", "16777215", CFG_PARAM_INT, __OFF(msteams_cache_ttl), NULL },
{ NULL, "lru.msteams.scope", "0", "0", "1", CFG_PARAM_INT, __OFF(msteams_cache_scope), clbk_only_with_global_ctx },


/* fpc dns cache */
{ NULL, "lru.fpc_dns.size", "1024", "0", "16777215", CFG_PARAM_INT, __OFF(fpc_dns_cache_num_entries), NULL },
{ NULL, "lru.fpc_dns.ttl", "60", "0", "16777215", CFG_PARAM_INT, __OFF(fpc_dns_cache_ttl), NULL },
{ NULL, "lru.fpc_dns.scope", "0", "0", "1", CFG_PARAM_INT, __OFF(fpc_dns_cache_scope), clbk_only_with_global_ctx },

{ NULL, NULL, NULL, NULL, NULL, 0, -1, NULL },
};

Expand Down
Loading