From 14a00a7a475e4106320f0d5ca19fe1b6a6459de0 Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Wed, 12 Apr 2023 14:05:43 +0200 Subject: [PATCH 01/50] test: flow's server address handling --- src/flow_checks/RareDestination.cpp | 42 +++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index faf814fe8b03..ad572569f405 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -22,16 +22,54 @@ #include "ntop_includes.h" #include "flow_checks_includes.h" +#define TODO_HERE 1 + /* ***************************************************** */ void RareDestination::protocolDetected(Flow *f) { bool is_rare_destination = false; + /* TODO: check if this is a real rare destination */ if(f->getFlowServerInfo() != NULL) { #ifdef TODO_HERE - ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare destination %s", f->getFlowServerInfo()); - is_rare_destination = true; + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare destination %s", f->getFlowServerInfo()); + + if (f->isLocalToLocal()) + { + u_int32_t key = 0; + Host * dest = f->get_srv_host(); + + if (dest->isDHCPHost()) + { + key = dest->getMac()->key(); + ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare destination MAC detected"); + } + + if (dest->isIPv6()) + { + const ndpi_in6_addr * destv6 = dest->get_ip()->get_ipv6(); + } + + if (dest->isIPv4()) + { + key = dest->get_ip()->get_ipv4(); + + } + + ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare destination IP %u", key); + } + + /* + Host * source = f->get_cli_host(); + Host * dest = f->get_srv_host(); + source->get_ip()->equal(inet_addr("192.168.43.247")); + dest->isDHCPHost() + u_int8_t mac = *(dest->get_mac()); + */ + + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Local Host Json %s", json_object_to_json_string_ext(source->get_ip()->getJSONObject(), JSON_C_TO_STRING_SPACED | JSON_C_TO_STRING_PRETTY)); + //is_rare_destination = true; #endif } From 931fe1b06df2aa9ba9cbbfaaae04e011e26fa6ed Mon Sep 17 00:00:00 2001 From: Leonardo Brugnano Date: Wed, 3 May 2023 16:07:23 +0200 Subject: [PATCH 02/50] First sketch --- src/flow_checks/RareDestination.cpp | 63 ++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index faf814fe8b03..399cd923c08f 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -21,6 +21,9 @@ #include "ntop_includes.h" #include "flow_checks_includes.h" +#include + +#define rareDestEpoch 100 /* placeholder value */ /* ***************************************************** */ @@ -28,11 +31,61 @@ void RareDestination::protocolDetected(Flow *f) { bool is_rare_destination = false; /* TODO: check if this is a real rare destination */ - if(f->getFlowServerInfo() != NULL) { -#ifdef TODO_HERE - ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare destination %s", f->getFlowServerInfo()); - is_rare_destination = true; -#endif + + /* + BMap is hosts bitmap + BDirty is dirty bit bitmap of BMap + -------- + time_t rareDestLastEpoch defined in class Host + -------- + struct { + bool ongoing; + time_t start; + time_t duration; + u_int32_t seen; + u_int32_t toSee; + + } training; -- defined in class host + -------- + ndpi_bitmap_xor(ndpi_bitmap*, ndpi_bitmap*) defined in ndpi_api.h + */ + + if(f->getFlowServerInfo() != NULL) + { + time_t timeNow = time(nullptr); + + /* to check if cardinality does what the right thing here */ + if (!ndpi_bitmap_cardinality(&BMap)) { + training->ongoing = true; + training->seen = 0; + training->start = timeNow; + } + if (training->ongoing && training->seen >= training->toSee && timeNow - training->start >= training->duration) + training->ongoing = false; + + if (!training->ongoing && timeNow - rareDestLastEpoch > rareDestEpoch) + { + if (timeNow - rareDestLastEpoch > 2*rareDestEpoch) + { + ndpi_bitmap_clear(&BMap); + ndpi_bitmap_clear(&BDirty); + } + else + { + ndpi_bitmap_xor(&BDirty, &BMap); // updates BDirty + ndpi_bitmap_and(&Bmap, & BDirty); // makes BMap = BDirty + } + rareDestLastEpoch = timeNow; + } + + u_int32_t hash = hashFun(f); // Yuriy's job + if (ndpi_bitmap_isset(&BMap, hash) + ndpi_bitmap_unset(&BDirty, hash); + else { + ndpi_bitmap_set(&BMap, hash); + if (!training->ongoing) is_rare_destination = true; + else training->seen++; + } } if(is_rare_destination) { From 233638f9930e978afe3b5f8b03b157987cceb074 Mon Sep 17 00:00:00 2001 From: Leonardo Brugnano Date: Wed, 3 May 2023 16:55:36 +0200 Subject: [PATCH 03/50] Second sketch --- src/flow_checks/RareDestination.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index 399cd923c08f..9b5b31a24925 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -21,7 +21,6 @@ #include "ntop_includes.h" #include "flow_checks_includes.h" -#include #define rareDestEpoch 100 /* placeholder value */ @@ -52,9 +51,8 @@ void RareDestination::protocolDetected(Flow *f) { if(f->getFlowServerInfo() != NULL) { - time_t timeNow = time(nullptr); + time_t timeNow = time(nullptr); // in time.h - /* to check if cardinality does what the right thing here */ if (!ndpi_bitmap_cardinality(&BMap)) { training->ongoing = true; training->seen = 0; @@ -63,9 +61,9 @@ void RareDestination::protocolDetected(Flow *f) { if (training->ongoing && training->seen >= training->toSee && timeNow - training->start >= training->duration) training->ongoing = false; - if (!training->ongoing && timeNow - rareDestLastEpoch > rareDestEpoch) + if (!training->ongoing && timeNow - rareDestLastEpoch >= rareDestEpoch) { - if (timeNow - rareDestLastEpoch > 2*rareDestEpoch) + if (timeNow - rareDestLastEpoch >= 2*rareDestEpoch) { ndpi_bitmap_clear(&BMap); ndpi_bitmap_clear(&BDirty); From 2c378d139fadd0943b07b8953eae90887afcbdf6 Mon Sep 17 00:00:00 2001 From: Zocco Giuseppe Date: Fri, 5 May 2023 10:00:21 +0200 Subject: [PATCH 04/50] Host extension for RareDest implementation. --- include/Host.h | 36 ++++++++++++++++++++++++++++++++++++ src/Host.cpp | 6 ++++++ 2 files changed, 42 insertions(+) diff --git a/include/Host.h b/include/Host.h index 48dc754facd8..fa6b01c7f4b4 100644 --- a/include/Host.h +++ b/include/Host.h @@ -68,6 +68,19 @@ class Host : public GenericHashEntry, public HostAlertableEntity, public Score, char *ssdpLocation; + /* Host data: update Host::RareDest implementation*/ + ndpi_bitmap *bMap; + ndpi_bitmap *bDirty; + time_t lastEpoch; + + struct { + bool ongoing; + time_t start; + time_t duration = DURATION_TRAINING; //in seconds + u_int32_t seen; + u_int32_t toSee = TO_SEE_TRAINING; // + } rareDestTraining; + /* END Host data: */ /* Counters used by host alerts */ @@ -695,6 +708,29 @@ class Host : public GenericHashEntry, public HostAlertableEntity, public Score, inline u_int16_t getNumContactedTCPUDPServerPortsNoTX() { return(tcp_udp_contacted_ports_no_tx ? (u_int16_t)ndpi_bitmap_cardinality(tcp_udp_contacted_ports_no_tx) : 0); } inline void setContactedTCPUDPServerPortNoTX(u_int16_t port) { if(tcp_udp_contacted_ports_no_tx) ndpi_bitmap_set(tcp_udp_contacted_ports_no_tx, port); } + /*RareDest Extension method*/ + //metodi bitmap + inline ndpi_bitmap* getBMap() const {return(bMap);} + inline ndpi_bitmap* getBDirty() const {return(bDirty);} + + inline time_t getRareDestLastEpoch() const {return(lastEpoch);} + inline void setRareDestLastEpoch(time_t t) {lastEpoch=t;} + + inline bool isOngoingRareDestTrainig() const {return(rareDestTraining.ongoing);} + inline void setOngoingRareDestTrainig(bool b) {rareDestTraining.ongoing = b;} + + inline time_t getStartRareDestTrainig() const {return(rareDestTraining.start);} + inline void setStartRareDestTrainig(time_t t) {rareDestTraining.start = t;} + + inline time_t getDurationRareDestTrainig() const {return(rareDestTraining.duration);} + + inline u_int32_t getSeenRareDestTrainig() const {return(rareDestTraining.seen);} + inline void clearSeenRareDestTrainig(u_int32_t n) {rareDestTraining.seen = 0;} + inline void incrementSeenRareDestTrainig(u_int32_t n) {rareDestTraining.seen = rareDestTrainig.seen++;} + + inline u_int32_t getToSeeRareDestTrainig() const {return(rareDestTraining.toSee);} + /**/ + void resetHostContacts(); }; diff --git a/src/Host.cpp b/src/Host.cpp index 3f5fab17ac98..c95bc4c495ab 100644 --- a/src/Host.cpp +++ b/src/Host.cpp @@ -108,6 +108,9 @@ Host::~Host() { if(externalAlert.msg) free(externalAlert.msg); if(tcp_udp_contacted_ports_no_tx) ndpi_bitmap_free(tcp_udp_contacted_ports_no_tx); + if(bMap) ndpi_bitmap_free(bMap); + if(bDirty) ndpi_bitmap_free(bDirty); + ndpi_hll_destroy(&outgoing_hosts_tcp_udp_port_with_no_tx_hll); ndpi_hll_destroy(&incoming_hosts_tcp_udp_port_with_no_tx_hll); } @@ -303,6 +306,9 @@ void Host::initialize(Mac *_mac, u_int16_t _vlanId, u_int16_t observation_point_ ndpi_hll_init(&outgoing_hosts_tcp_udp_port_with_no_tx_hll, 5 /* StdError: 18.4% */); ndpi_hll_init(&incoming_hosts_tcp_udp_port_with_no_tx_hll, 5 /* StdError: 18.4% */); + bMap = ndpi_bitmap_alloc(); + bDirty = ndpi_bitmap_alloc(); + deferredInitialization(); /* TODO To be called asynchronously for improving performance */ } From 880d2213c6d182bc130dbfadb959817c4ff457b4 Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Fri, 5 May 2023 18:14:21 +0200 Subject: [PATCH 05/50] refactor: polished hashing section --- src/flow_checks/RareDestination.cpp | 84 +++++++++++++++++------------ 1 file changed, 50 insertions(+), 34 deletions(-) diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index ad572569f405..cbe2b4ac114b 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -26,49 +26,65 @@ /* ***************************************************** */ +bool getDestinationHash(Flow *f, u_int32_t *hash) { + Host * dest = f->get_srv_host(); + if (f->isLocalToLocal()) + { + char buf[64]; + if (!dest->isMulticastHost() && dest->isDHCPHost()) + { + char *mac = dest->getMac()->print(buf,sizeof(buf)); + *hash = Utils::hashString(mac); + /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination MAC %u - %s", *hash, mac); */ + } + + if (dest->isIPv6()) + { + char *ipv6 = dest->get_ip()->print(buf,sizeof(buf)); + *hash = Utils::hashString(ipv6); + /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination IPv6 %u - %s", *hash, ipv6); */ + } + + if (dest->isIPv4()) + { + char *ipv4 = dest->get_ip()->print(buf,sizeof(buf)); + *hash = Utils::hashString(ipv4); + /* u_int32_t quick_key = ndpi_quick_hash((unsigned char *)ipv4,sizeof(ipv4)); */ + /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination IPv4 %u - %s", *hash, ipv4); */ + } + } + + if (f->isLocalToRemote()) + { + char name_buf[128]; + char *domain = dest->get_name(name_buf, sizeof(name_buf), false); + *hash = Utils::hashString(domain); + /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare remote destination %u - %s", *hash, domain); */ + } + + if ( *hash == 0 && (!f->isLocalToLocal() || !f->isLocalToRemote()) ) + { + /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Destination not hashed: %s", json_object_to_json_string(dest->get_ip()->getJSONObject())); */ + return (false); + } + + return (true); +} + +/* ***************************************************** */ + void RareDestination::protocolDetected(Flow *f) { bool is_rare_destination = false; - /* TODO: check if this is a real rare destination */ if(f->getFlowServerInfo() != NULL) { #ifdef TODO_HERE - //ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare destination %s", f->getFlowServerInfo()); + u_int32_t hash = 0; + if(!getDestinationHash(f,&hash)) { return; } + ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare destination hash %u", hash); - if (f->isLocalToLocal()) - { - u_int32_t key = 0; - Host * dest = f->get_srv_host(); - - if (dest->isDHCPHost()) - { - key = dest->getMac()->key(); - ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare destination MAC detected"); - } - - if (dest->isIPv6()) - { - const ndpi_in6_addr * destv6 = dest->get_ip()->get_ipv6(); - } - - if (dest->isIPv4()) - { - key = dest->get_ip()->get_ipv4(); - - } - - ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare destination IP %u", key); - } - /* - Host * source = f->get_cli_host(); - Host * dest = f->get_srv_host(); - source->get_ip()->equal(inet_addr("192.168.43.247")); - dest->isDHCPHost() - u_int8_t mac = *(dest->get_mac()); - */ - //ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Local Host Json %s", json_object_to_json_string_ext(source->get_ip()->getJSONObject(), JSON_C_TO_STRING_SPACED | JSON_C_TO_STRING_PRETTY)); //is_rare_destination = true; #endif } From b005bbbabba7d1bc2ade1d61970d20127b5f4410 Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Sat, 6 May 2023 01:04:51 +0200 Subject: [PATCH 06/50] fix: fix typo & set dummy default params --- include/Host.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/Host.h b/include/Host.h index fa6b01c7f4b4..787e114c83ad 100644 --- a/include/Host.h +++ b/include/Host.h @@ -76,9 +76,9 @@ class Host : public GenericHashEntry, public HostAlertableEntity, public Score, struct { bool ongoing; time_t start; - time_t duration = DURATION_TRAINING; //in seconds + time_t duration = 600; //sec u_int32_t seen; - u_int32_t toSee = TO_SEE_TRAINING; // + u_int32_t toSee = 100; // # of Flows } rareDestTraining; /* END Host data: */ @@ -726,7 +726,7 @@ class Host : public GenericHashEntry, public HostAlertableEntity, public Score, inline u_int32_t getSeenRareDestTrainig() const {return(rareDestTraining.seen);} inline void clearSeenRareDestTrainig(u_int32_t n) {rareDestTraining.seen = 0;} - inline void incrementSeenRareDestTrainig(u_int32_t n) {rareDestTraining.seen = rareDestTrainig.seen++;} + inline void incrementSeenRareDestTrainig(u_int32_t n) {rareDestTraining.seen = rareDestTraining.seen++;} inline u_int32_t getToSeeRareDestTrainig() const {return(rareDestTraining.toSee);} /**/ From 3f32c1993a144568fbc803dd2dba944d8c63cb67 Mon Sep 17 00:00:00 2001 From: Leonardo Brugnano Date: Sun, 7 May 2023 10:25:13 +0200 Subject: [PATCH 07/50] API adaption to new Host class --- src/flow_checks/RareDestination.cpp | 62 +++++++++++------------------ 1 file changed, 24 insertions(+), 38 deletions(-) diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index 9b5b31a24925..ad4e1ae0deea 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -25,63 +25,49 @@ #define rareDestEpoch 100 /* placeholder value */ /* ***************************************************** */ +/* + ndpi_bitmap_xor(ndpi_bitmap*, ndpi_bitmap*) defined in ndpi_api.h +*/ void RareDestination::protocolDetected(Flow *f) { bool is_rare_destination = false; - /* TODO: check if this is a real rare destination */ - - /* - BMap is hosts bitmap - BDirty is dirty bit bitmap of BMap - -------- - time_t rareDestLastEpoch defined in class Host - -------- - struct { - bool ongoing; - time_t start; - time_t duration; - u_int32_t seen; - u_int32_t toSee; - - } training; -- defined in class host - -------- - ndpi_bitmap_xor(ndpi_bitmap*, ndpi_bitmap*) defined in ndpi_api.h - */ - if(f->getFlowServerInfo() != NULL) { - time_t timeNow = time(nullptr); // in time.h + time_t timeNow = time(nullptr); + Host *cliHost = f->get_cli_host(); + ndpi_bitmap *bMap = cliHost->getBMap(); + ndpi_bitmap *bDirty = cliHost->getBDirty(); - if (!ndpi_bitmap_cardinality(&BMap)) { - training->ongoing = true; - training->seen = 0; - training->start = timeNow; + if (!ndpi_bitmap_cardinality(bMap)) { + cliHost->setOngoingRareDestTraining(true); + cliHost->clearSeenRareDestTraining(); + cliHost->setStartRareDestTraining(timeNow); } - if (training->ongoing && training->seen >= training->toSee && timeNow - training->start >= training->duration) - training->ongoing = false; + if (cliHost->isOngoingRareDestTraining() && cliHost->getSeenRareDestTraining() >= cliHost->getToSeeRareDestTraining() && timeNow - cliHost->getStartRareDestTraining() >= cliHost->getDurationRareDestTraining()) + cliHost->setOngoingRareDestTraining(false); - if (!training->ongoing && timeNow - rareDestLastEpoch >= rareDestEpoch) + if (!cliHost->isOngoingRareDestTraining() && timeNow - cliHost->getRareDestLastEpoch() >= rareDestEpoch) { - if (timeNow - rareDestLastEpoch >= 2*rareDestEpoch) + if (timeNow - cliHost->getRareDestLastEpoch() >= 2*rareDestEpoch) { - ndpi_bitmap_clear(&BMap); - ndpi_bitmap_clear(&BDirty); + ndpi_bitmap_clear(bMap); + ndpi_bitmap_clear(bDirty); } else { - ndpi_bitmap_xor(&BDirty, &BMap); // updates BDirty - ndpi_bitmap_and(&Bmap, & BDirty); // makes BMap = BDirty + ndpi_bitmap_xor(bDirty, bMap); // updates BDirty + ndpi_bitmap_and(bMap, bDirty); // makes BMap = BDirty } - rareDestLastEpoch = timeNow; + cliHost->setRareDestLastEpoch(timeNow); } u_int32_t hash = hashFun(f); // Yuriy's job - if (ndpi_bitmap_isset(&BMap, hash) - ndpi_bitmap_unset(&BDirty, hash); + if (ndpi_bitmap_isset(bMap, hash) + ndpi_bitmap_unset(bDirty, hash); else { - ndpi_bitmap_set(&BMap, hash); - if (!training->ongoing) is_rare_destination = true; + ndpi_bitmap_set(bMap, hash); + if (!cliHost->isOngoingRareDestTraining()) is_rare_destination = true; else training->seen++; } } From 98e92ed0e912de37986b10dc51600dc8af3bf503 Mon Sep 17 00:00:00 2001 From: Leonardo Brugnano Date: Sun, 7 May 2023 10:30:05 +0200 Subject: [PATCH 08/50] [fix] API adaption to new Host class --- src/flow_checks/RareDestination.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index ad4e1ae0deea..b480663eaebe 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -68,7 +68,7 @@ void RareDestination::protocolDetected(Flow *f) { else { ndpi_bitmap_set(bMap, hash); if (!cliHost->isOngoingRareDestTraining()) is_rare_destination = true; - else training->seen++; + else cliHost->incrementSeenRareDestTrainig(); } } From e2278e4cacc789f8dd378fe6891b2a8f9e9e2795 Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Mon, 8 May 2023 01:51:58 +0200 Subject: [PATCH 09/50] features & refactor & fixes: fixed two RareDestTrainig's methods. renamed bitmaps ( more concise with its functionality ). added serialization to redis: ( dumpRareDestToRedis and loadRareDestFromRedis methods ) --- include/Host.h | 44 +++++++++--------- include/ntop_defines.h | 6 +++ src/Host.cpp | 103 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 128 insertions(+), 25 deletions(-) diff --git a/include/Host.h b/include/Host.h index 787e114c83ad..f581657bb4e8 100644 --- a/include/Host.h +++ b/include/Host.h @@ -69,16 +69,16 @@ class Host : public GenericHashEntry, public HostAlertableEntity, public Score, char *ssdpLocation; /* Host data: update Host::RareDest implementation*/ - ndpi_bitmap *bMap; - ndpi_bitmap *bDirty; - time_t lastEpoch; + ndpi_bitmap *rare_dest; + ndpi_bitmap *rare_dest_revise; + time_t last_epoch; struct { bool ongoing; time_t start; - time_t duration = 600; //sec + time_t duration = RARE_DEST_DURATION_TRAINING; u_int32_t seen; - u_int32_t toSee = 100; // # of Flows + u_int32_t toSee = RARE_DEST_FLOWS_TO_SEE_TRAINING; } rareDestTraining; /* END Host data: */ @@ -708,28 +708,30 @@ class Host : public GenericHashEntry, public HostAlertableEntity, public Score, inline u_int16_t getNumContactedTCPUDPServerPortsNoTX() { return(tcp_udp_contacted_ports_no_tx ? (u_int16_t)ndpi_bitmap_cardinality(tcp_udp_contacted_ports_no_tx) : 0); } inline void setContactedTCPUDPServerPortNoTX(u_int16_t port) { if(tcp_udp_contacted_ports_no_tx) ndpi_bitmap_set(tcp_udp_contacted_ports_no_tx, port); } - /*RareDest Extension method*/ - //metodi bitmap - inline ndpi_bitmap* getBMap() const {return(bMap);} - inline ndpi_bitmap* getBDirty() const {return(bDirty);} + /* RareDest Extension methods */ + + inline ndpi_bitmap* getRareDestBMap() const { return(rare_dest); } + inline ndpi_bitmap* getRareDestReviseBMap() const { return(rare_dest_revise); } - inline time_t getRareDestLastEpoch() const {return(lastEpoch);} - inline void setRareDestLastEpoch(time_t t) {lastEpoch=t;} + inline time_t getRareDestLastEpoch() const { return(last_epoch); } + inline void setRareDestLastEpoch(time_t t) { last_epoch=t; } - inline bool isOngoingRareDestTrainig() const {return(rareDestTraining.ongoing);} - inline void setOngoingRareDestTrainig(bool b) {rareDestTraining.ongoing = b;} + inline bool isOngoingRareDestTrainig() const { return(rareDestTraining.ongoing); } + inline void setOngoingRareDestTrainig(bool b) { rareDestTraining.ongoing = b; } - inline time_t getStartRareDestTrainig() const {return(rareDestTraining.start);} - inline void setStartRareDestTrainig(time_t t) {rareDestTraining.start = t;} + inline time_t getStartRareDestTrainig() const { return(rareDestTraining.start); } + inline void setStartRareDestTrainig(time_t t) { rareDestTraining.start = t; } - inline time_t getDurationRareDestTrainig() const {return(rareDestTraining.duration);} + inline time_t getDurationRareDestTrainig() const { return(rareDestTraining.duration); } - inline u_int32_t getSeenRareDestTrainig() const {return(rareDestTraining.seen);} - inline void clearSeenRareDestTrainig(u_int32_t n) {rareDestTraining.seen = 0;} - inline void incrementSeenRareDestTrainig(u_int32_t n) {rareDestTraining.seen = rareDestTraining.seen++;} + inline u_int32_t getSeenRareDestTrainig() const { return(rareDestTraining.seen); } + inline void clearSeenRareDestTrainig() { rareDestTraining.seen = 0; } + inline void incrementSeenRareDestTrainig() { rareDestTraining.seen++; } - inline u_int32_t getToSeeRareDestTrainig() const {return(rareDestTraining.toSee);} - /**/ + inline u_int32_t getToSeeRareDestTrainig() const { return(rareDestTraining.toSee); } + + void dumpRareDestToRedis(); + bool loadRareDestFromRedis(); void resetHostContacts(); }; diff --git a/include/ntop_defines.h b/include/ntop_defines.h index 8c2bd9754257..0fe51ee5ea97 100644 --- a/include/ntop_defines.h +++ b/include/ntop_defines.h @@ -1306,4 +1306,10 @@ extern struct ntopngLuaContext* getUserdata(struct lua_State *vm); /******************************************************************************/ +#define HOST_RARE_DEST_SERIALIZED_KEY "ntopng.serialized_host_rare_dest_fields.ifid_%u" +#define RARE_DEST_DURATION_TRAINING 3600 /* seconds */ +#define RARE_DEST_FLOWS_TO_SEE_TRAINING 100 + +/******************************************************************************/ + #endif /* _NTOP_DEFINES_H_ */ diff --git a/src/Host.cpp b/src/Host.cpp index c95bc4c495ab..345071d73c13 100644 --- a/src/Host.cpp +++ b/src/Host.cpp @@ -108,8 +108,9 @@ Host::~Host() { if(externalAlert.msg) free(externalAlert.msg); if(tcp_udp_contacted_ports_no_tx) ndpi_bitmap_free(tcp_udp_contacted_ports_no_tx); - if(bMap) ndpi_bitmap_free(bMap); - if(bDirty) ndpi_bitmap_free(bDirty); + dumpRareDestToRedis(); + if(rare_dest) ndpi_bitmap_free(rare_dest); + if(rare_dest_revise) ndpi_bitmap_free(rare_dest_revise); ndpi_hll_destroy(&outgoing_hosts_tcp_udp_port_with_no_tx_hll); ndpi_hll_destroy(&incoming_hosts_tcp_udp_port_with_no_tx_hll); @@ -306,8 +307,11 @@ void Host::initialize(Mac *_mac, u_int16_t _vlanId, u_int16_t observation_point_ ndpi_hll_init(&outgoing_hosts_tcp_udp_port_with_no_tx_hll, 5 /* StdError: 18.4% */); ndpi_hll_init(&incoming_hosts_tcp_udp_port_with_no_tx_hll, 5 /* StdError: 18.4% */); - bMap = ndpi_bitmap_alloc(); - bDirty = ndpi_bitmap_alloc(); + if( !loadRareDestFromRedis() ){ + last_epoch = 0; + rare_dest = ndpi_bitmap_alloc(); + rare_dest_revise = ndpi_bitmap_alloc(); + } deferredInitialization(); /* TODO To be called asynchronously for improving performance */ } @@ -2590,3 +2594,94 @@ void Host::resetHostContacts() { ndpi_hll_reset(&incoming_hosts_tcp_udp_port_with_no_tx_hll); ndpi_bitmap_clear(tcp_udp_contacted_ports_no_tx); } + +/* *************************************** */ + +void Host::dumpRareDestToRedis() { + char key[CONST_MAX_LEN_REDIS_KEY], buf[32], last_epoch_ser[32], size_ser[32], *value; + Redis *redis = ntop->getRedis(); + size_t size; + + if((!redis) || (!iface) || (!rare_dest) || (!rare_dest_revise)) return; + + snprintf(key, sizeof(key), HOST_RARE_DEST_SERIALIZED_KEY, iface->get_id()); + + snprintf(buf, sizeof(buf), "rare_dest"); + size = ndpi_bitmap_serialize(rare_dest, &value); + if (value) { + char *encoded_bmap = Utils::base64_encode((unsigned char *)value, size); + size = strlen(encoded_bmap)+2; + redis->hashSet(key, buf, encoded_bmap); + free(value); + } + + snprintf(buf, sizeof(buf), "rare_dest_len"); + snprintf(size_ser, sizeof(size_ser), "%lu", size); + redis->hashSet(key, buf, size_ser); + + snprintf(buf, sizeof(buf), "rare_dest_revise"); + size = ndpi_bitmap_serialize(rare_dest_revise, &value); + + if (value) { + char *encoded_bmap = Utils::base64_encode((unsigned char *)value, size); + size = strlen(encoded_bmap)+2; + redis->hashSet(key, buf, value); + free(value); + } + + snprintf(buf, sizeof(buf), "rare_dest_revise_len"); + snprintf(size_ser, sizeof(size_ser), "%lu", size); + redis->hashSet(key, buf, size_ser); + + + snprintf(buf, sizeof(buf), "last_epoch"); + snprintf(last_epoch_ser, sizeof(last_epoch_ser), "%ld", last_epoch); + + redis->hashSet(key, buf, last_epoch_ser); +} + +bool Host::loadRareDestFromRedis() { + char key[CONST_MAX_LEN_REDIS_KEY], buf[32], last_epoch_str[32], size_str[32], *value; + Redis *redis = ntop->getRedis(); + size_t size; + + if((!redis) || (!iface)) return(false); + + snprintf(key, sizeof(key), HOST_RARE_DEST_SERIALIZED_KEY, iface->get_id()); + + snprintf(buf, sizeof(buf), "last_epoch"); + if( redis->hashGet(key, buf, last_epoch_str, sizeof(last_epoch_str)) != 0 ) return(false); + last_epoch = atol(last_epoch_str); + + snprintf(buf, sizeof(buf), "rare_dest_len"); + if( redis->hashGet(key, buf, size_str, sizeof(size_str)) != 0 ) return(false); + size = (size_t)strtoul(size_str, NULL, 10); + + if((value = (char *) malloc(size)) == NULL) { + ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to allocate memory to deserialize %s", key); + return(false); + } + + snprintf(buf, sizeof(buf), "rare_dest"); + if(redis->hashGet(key, buf, value, size) == 0) { + rare_dest = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); + } + free(value); + + snprintf(buf, sizeof(buf), "rare_dest_revise_len"); + if( redis->hashGet(key, buf, size_str, sizeof(size_str)) != 0 ) return(false); + size = (size_t)strtoul(size_str, NULL, 10); + + if((value = (char *) malloc(size)) == NULL) { + ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to allocate memory to deserialize %s", key); + return(false); + } + + snprintf(buf, sizeof(buf), "rare_dest_revise"); + if(redis->hashGet(key, buf, value, size) == 0) { + rare_dest_revise = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); + } + free(value); + + return(true); +} From 1802dc2b03b0e9235d28a4a28946690f9af5fe03 Mon Sep 17 00:00:00 2001 From: Leonardo Brugnano Date: Mon, 8 May 2023 09:32:09 +0200 Subject: [PATCH 10/50] Variables renaming and hashing implementation --- src/flow_checks/RareDestination.cpp | 33 +++++++++++++++++------------ 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index b480663eaebe..3f369ee5aa76 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -36,10 +36,10 @@ void RareDestination::protocolDetected(Flow *f) { { time_t timeNow = time(nullptr); Host *cliHost = f->get_cli_host(); - ndpi_bitmap *bMap = cliHost->getBMap(); - ndpi_bitmap *bDirty = cliHost->getBDirty(); + ndpi_bitmap *rare_dest = cliHost->getRareDestBMap(); + ndpi_bitmap *rare_dest_revise = cliHost->getRareDestReviseBMap(); - if (!ndpi_bitmap_cardinality(bMap)) { + if (!ndpi_bitmap_cardinality(rare_dest)) { cliHost->setOngoingRareDestTraining(true); cliHost->clearSeenRareDestTraining(); cliHost->setStartRareDestTraining(timeNow); @@ -51,25 +51,30 @@ void RareDestination::protocolDetected(Flow *f) { { if (timeNow - cliHost->getRareDestLastEpoch() >= 2*rareDestEpoch) { - ndpi_bitmap_clear(bMap); - ndpi_bitmap_clear(bDirty); + ndpi_bitmap_clear(rare_dest); + ndpi_bitmap_clear(rare_dest_revise); } else { - ndpi_bitmap_xor(bDirty, bMap); // updates BDirty - ndpi_bitmap_and(bMap, bDirty); // makes BMap = BDirty + ndpi_bitmap_xor(rare_dest_revise, rare_dest); // updates rare_dest_revise + ndpi_bitmap_and(rare_dest, rare_dest_revise); // makes rare_dest = rare_dest_revise } cliHost->setRareDestLastEpoch(timeNow); } - u_int32_t hash = hashFun(f); // Yuriy's job - if (ndpi_bitmap_isset(bMap, hash) - ndpi_bitmap_unset(bDirty, hash); - else { - ndpi_bitmap_set(bMap, hash); - if (!cliHost->isOngoingRareDestTraining()) is_rare_destination = true; - else cliHost->incrementSeenRareDestTrainig(); + u_int32_t hash; + if (getDestinationHash(f, &hash)) + { + if (ndpi_bitmap_isset(rare_dest, hash) + ndpi_bitmap_unset(rare_dest_revise, hash); + else { + ndpi_bitmap_set(rare_dest, hash); + if (!cliHost->isOngoingRareDestTraining()) is_rare_destination = true; + else cliHost->incrementSeenRareDestTrainig(); + } } + else + /* error: hashing not possible */ } if(is_rare_destination) { From f9ccc389be0560ad8c1d6e3d696d0997c7907116 Mon Sep 17 00:00:00 2001 From: Leonardo Brugnano Date: Mon, 8 May 2023 09:37:52 +0200 Subject: [PATCH 11/50] Fix typo --- include/Host.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/include/Host.h b/include/Host.h index f581657bb4e8..396b7175cb1a 100644 --- a/include/Host.h +++ b/include/Host.h @@ -716,19 +716,19 @@ class Host : public GenericHashEntry, public HostAlertableEntity, public Score, inline time_t getRareDestLastEpoch() const { return(last_epoch); } inline void setRareDestLastEpoch(time_t t) { last_epoch=t; } - inline bool isOngoingRareDestTrainig() const { return(rareDestTraining.ongoing); } - inline void setOngoingRareDestTrainig(bool b) { rareDestTraining.ongoing = b; } + inline bool isOngoingRareDestTraining() const { return(rareDestTraining.ongoing); } + inline void setOngoingRareDestTraining(bool b) { rareDestTraining.ongoing = b; } - inline time_t getStartRareDestTrainig() const { return(rareDestTraining.start); } - inline void setStartRareDestTrainig(time_t t) { rareDestTraining.start = t; } + inline time_t getStartRareDestTraining() const { return(rareDestTraining.start); } + inline void setStartRareDestTraining(time_t t) { rareDestTraining.start = t; } - inline time_t getDurationRareDestTrainig() const { return(rareDestTraining.duration); } + inline time_t getDurationRareDestTraining() const { return(rareDestTraining.duration); } - inline u_int32_t getSeenRareDestTrainig() const { return(rareDestTraining.seen); } - inline void clearSeenRareDestTrainig() { rareDestTraining.seen = 0; } - inline void incrementSeenRareDestTrainig() { rareDestTraining.seen++; } + inline u_int32_t getSeenRareDestTraining() const { return(rareDestTraining.seen); } + inline void clearSeenRareDestTraining() { rareDestTraining.seen = 0; } + inline void incrementSeenRareDestTraining() { rareDestTraining.seen++; } - inline u_int32_t getToSeeRareDestTrainig() const { return(rareDestTraining.toSee); } + inline u_int32_t getToSeeRareDestTraining() const { return(rareDestTraining.toSee); } void dumpRareDestToRedis(); bool loadRareDestFromRedis(); From a626af3116cf32bc9710385b364c514fa359925a Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Mon, 8 May 2023 11:37:45 +0200 Subject: [PATCH 12/50] bugfix: correct memory leak --- src/Host.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Host.cpp b/src/Host.cpp index 345071d73c13..6753747ce4e0 100644 --- a/src/Host.cpp +++ b/src/Host.cpp @@ -2612,6 +2612,7 @@ void Host::dumpRareDestToRedis() { char *encoded_bmap = Utils::base64_encode((unsigned char *)value, size); size = strlen(encoded_bmap)+2; redis->hashSet(key, buf, encoded_bmap); + free(encoded_bmap); free(value); } @@ -2625,8 +2626,9 @@ void Host::dumpRareDestToRedis() { if (value) { char *encoded_bmap = Utils::base64_encode((unsigned char *)value, size); size = strlen(encoded_bmap)+2; - redis->hashSet(key, buf, value); - free(value); + redis->hashSet(key, buf, encoded_bmap); + free(encoded_bmap); + free(value); } snprintf(buf, sizeof(buf), "rare_dest_revise_len"); @@ -2638,6 +2640,7 @@ void Host::dumpRareDestToRedis() { snprintf(last_epoch_ser, sizeof(last_epoch_ser), "%ld", last_epoch); redis->hashSet(key, buf, last_epoch_ser); + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Dump to redis finished"); } bool Host::loadRareDestFromRedis() { From 66a3fa8b52a93ed24244b9a7e396190882291018 Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Mon, 8 May 2023 11:59:25 +0200 Subject: [PATCH 13/50] fix: fixed typos --- src/flow_checks/RareDestination.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index 3f369ee5aa76..318eda75dc02 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -73,8 +73,7 @@ void RareDestination::protocolDetected(Flow *f) { else cliHost->incrementSeenRareDestTrainig(); } } - else - /* error: hashing not possible */ + /* error: hashing not possible */ } if(is_rare_destination) { From eb1d0e306ccfcc641ea527dd4a99412d09f87dd0 Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk <45283261+DropB1t@users.noreply.github.com> Date: Mon, 8 May 2023 12:11:57 +0200 Subject: [PATCH 14/50] Hashing edit (#3) * Edit suggestion * refactor: change position of curly braces --------- Co-authored-by: Leonardo Brugnano --- src/flow_checks/RareDestination.cpp | 39 ++++++++--------------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index cbe2b4ac114b..ff59e6616e66 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -28,47 +28,30 @@ bool getDestinationHash(Flow *f, u_int32_t *hash) { Host * dest = f->get_srv_host(); - if (f->isLocalToLocal()) - { + if (f->isLocalToLocal()) { char buf[64]; - if (!dest->isMulticastHost() && dest->isDHCPHost()) - { + if (!dest->isMulticastHost() && dest->isDHCPHost()) { char *mac = dest->getMac()->print(buf,sizeof(buf)); *hash = Utils::hashString(mac); /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination MAC %u - %s", *hash, mac); */ + return (true); } - if (dest->isIPv6()) - { - char *ipv6 = dest->get_ip()->print(buf,sizeof(buf)); - *hash = Utils::hashString(ipv6); - /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination IPv6 %u - %s", *hash, ipv6); */ - } - - if (dest->isIPv4()) - { - char *ipv4 = dest->get_ip()->print(buf,sizeof(buf)); - *hash = Utils::hashString(ipv4); - /* u_int32_t quick_key = ndpi_quick_hash((unsigned char *)ipv4,sizeof(ipv4)); */ - /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination IPv4 %u - %s", *hash, ipv4); */ + if (dest->isIPv6() || dest->isIPv4()) { + char *ip = dest->get_ip()->print(buf,sizeof(buf)); + *hash = Utils::hashString(ip); + /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination IPv6/IPv4 %u - %s", *hash, ip); */ + return (true); } } - - if (f->isLocalToRemote()) - { + if (f->isLocalToRemote()) { char name_buf[128]; char *domain = dest->get_name(name_buf, sizeof(name_buf), false); *hash = Utils::hashString(domain); /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare remote destination %u - %s", *hash, domain); */ + return (true); } - - if ( *hash == 0 && (!f->isLocalToLocal() || !f->isLocalToRemote()) ) - { - /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Destination not hashed: %s", json_object_to_json_string(dest->get_ip()->getJSONObject())); */ - return (false); - } - - return (true); + return (false); } /* ***************************************************** */ From 410ad4159f59df414e513fd840f6f6ffaf777ee6 Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Mon, 8 May 2023 17:58:35 +0200 Subject: [PATCH 15/50] refactor: refactor naming convention, defined constant values instead of struct's values --- include/Host.h | 11 +++++--- include/ntop_defines.h | 1 + src/flow_checks/RareDestination.cpp | 40 ++++++++++++++--------------- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/include/Host.h b/include/Host.h index 396b7175cb1a..92e581585fdb 100644 --- a/include/Host.h +++ b/include/Host.h @@ -76,9 +76,11 @@ class Host : public GenericHashEntry, public HostAlertableEntity, public Score, struct { bool ongoing; time_t start; - time_t duration = RARE_DEST_DURATION_TRAINING; u_int32_t seen; + /* + time_t duration = RARE_DEST_DURATION_TRAINING; u_int32_t toSee = RARE_DEST_FLOWS_TO_SEE_TRAINING; + */ } rareDestTraining; /* END Host data: */ @@ -722,13 +724,14 @@ class Host : public GenericHashEntry, public HostAlertableEntity, public Score, inline time_t getStartRareDestTraining() const { return(rareDestTraining.start); } inline void setStartRareDestTraining(time_t t) { rareDestTraining.start = t; } - inline time_t getDurationRareDestTraining() const { return(rareDestTraining.duration); } - inline u_int32_t getSeenRareDestTraining() const { return(rareDestTraining.seen); } inline void clearSeenRareDestTraining() { rareDestTraining.seen = 0; } - inline void incrementSeenRareDestTraining() { rareDestTraining.seen++; } + inline void incrementSeenRareDestTraining() { rareDestTraining.seen++; } + /* + inline time_t getDurationRareDestTraining() const { return(rareDestTraining.duration); } inline u_int32_t getToSeeRareDestTraining() const { return(rareDestTraining.toSee); } + */ void dumpRareDestToRedis(); bool loadRareDestFromRedis(); diff --git a/include/ntop_defines.h b/include/ntop_defines.h index 0fe51ee5ea97..de54386e72b9 100644 --- a/include/ntop_defines.h +++ b/include/ntop_defines.h @@ -1309,6 +1309,7 @@ extern struct ntopngLuaContext* getUserdata(struct lua_State *vm); #define HOST_RARE_DEST_SERIALIZED_KEY "ntopng.serialized_host_rare_dest_fields.ifid_%u" #define RARE_DEST_DURATION_TRAINING 3600 /* seconds */ #define RARE_DEST_FLOWS_TO_SEE_TRAINING 100 +#define RARE_DEST_EPOCH_DURATION 3600 /* seconds */ /******************************************************************************/ diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index 5dc0186c9278..46649c6dbe61 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -22,9 +22,6 @@ #include "ntop_includes.h" #include "flow_checks_includes.h" -#define TODO_HERE 1 -#define rareDestEpoch 100 /* placeholder value */ - /* ***************************************************** */ bool getDestinationHash(Flow *f, u_int32_t *hash) { @@ -66,48 +63,49 @@ void RareDestination::protocolDetected(Flow *f) { /* TODO: check if this is a real rare destination */ if(f->getFlowServerInfo() != NULL) { -#ifdef TODO_HERE u_int32_t hash = 0; if(!getDestinationHash(f,&hash)) { return; } - ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare destination hash %u", hash); + /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare destination hash %u", hash); */ - time_t timeNow = time(nullptr); - Host *cliHost = f->get_cli_host(); - ndpi_bitmap *rare_dest = cliHost->getRareDestBMap(); - ndpi_bitmap *rare_dest_revise = cliHost->getRareDestReviseBMap(); + Host *cli_host = f->get_cli_host(); + ndpi_bitmap *rare_dest = cli_host->getRareDestBMap(); + ndpi_bitmap *rare_dest_revise = cli_host->getRareDestReviseBMap(); + time_t t_now = time(NULL); if (!ndpi_bitmap_cardinality(rare_dest)) { - cliHost->setOngoingRareDestTraining(true); - cliHost->clearSeenRareDestTraining(); - cliHost->setStartRareDestTraining(timeNow); + cli_host->clearSeenRareDestTraining(); + cli_host->setOngoingRareDestTraining(true); + cli_host->setStartRareDestTraining(t_now); } - if (cliHost->isOngoingRareDestTraining() && cliHost->getSeenRareDestTraining() >= cliHost->getToSeeRareDestTraining() && timeNow - cliHost->getStartRareDestTraining() >= cliHost->getDurationRareDestTraining()) - cliHost->setOngoingRareDestTraining(false); - if (!cliHost->isOngoingRareDestTraining() && timeNow - cliHost->getRareDestLastEpoch() >= rareDestEpoch) { - if (timeNow - cliHost->getRareDestLastEpoch() >= 2*rareDestEpoch) { + if ( cli_host->isOngoingRareDestTraining() + && cli_host->getSeenRareDestTraining() >= RARE_DEST_FLOWS_TO_SEE_TRAINING + && t_now - cli_host->getStartRareDestTraining() >= RARE_DEST_DURATION_TRAINING ) + cli_host->setOngoingRareDestTraining(false); + + if (!cli_host->isOngoingRareDestTraining() && t_now - cli_host->getRareDestLastEpoch() >= RARE_DEST_EPOCH_DURATION) { + if (t_now - cli_host->getRareDestLastEpoch() >= 2*RARE_DEST_EPOCH_DURATION) { ndpi_bitmap_clear(rare_dest); ndpi_bitmap_clear(rare_dest_revise); } else { ndpi_bitmap_xor(rare_dest_revise, rare_dest); // updates rare_dest_revise ndpi_bitmap_and(rare_dest, rare_dest_revise); // makes rare_dest = rare_dest_revise } - cliHost->setRareDestLastEpoch(timeNow); + cli_host->setRareDestLastEpoch(t_now); } if (!hash) { - if (ndpi_bitmap_isset(rare_dest, hash) + if (ndpi_bitmap_isset(rare_dest, hash)) ndpi_bitmap_unset(rare_dest_revise, hash); else { ndpi_bitmap_set(rare_dest, hash); - if (!cliHost->isOngoingRareDestTraining()) is_rare_destination = true; - else cliHost->incrementSeenRareDestTrainig(); + if (!cli_host->isOngoingRareDestTraining()) is_rare_destination = true; + else cli_host->incrementSeenRareDestTraining(); } } /* error: hashing not possible */ //is_rare_destination = true; -#endif } if(is_rare_destination) { From fdc6d10bfd64aef79c012ac75ee98b2c8476c884 Mon Sep 17 00:00:00 2001 From: Leonardo Brugnano Date: Wed, 10 May 2023 09:17:11 +0200 Subject: [PATCH 16/50] RareDestination.cpp refactoring --- src/flow_checks/RareDestination.cpp | 58 ++++++++++++++++------------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index 46649c6dbe61..933758af9e0f 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -25,9 +25,9 @@ /* ***************************************************** */ bool getDestinationHash(Flow *f, u_int32_t *hash) { - Host * dest = f->get_srv_host(); + Host *dest = f->get_srv_host(); if (f->isLocalToLocal()) { - char buf[64]; + char buf[128]; if (!dest->isMulticastHost() && dest->isDHCPHost()) { char *mac = dest->getMac()->print(buf,sizeof(buf)); *hash = Utils::hashString(mac); @@ -55,15 +55,16 @@ bool getDestinationHash(Flow *f, u_int32_t *hash) { /* ***************************************************** */ /* - ndpi_bitmap_xor(ndpi_bitmap*, ndpi_bitmap*) defined in ndpi_api.h + ndpi_bitmap_xor(ndpi_bitmap*, ndpi_bitmap*) defined in ndpi_api.h, ndpi_bitmap.c */ void RareDestination::protocolDetected(Flow *f) { + bool is_rare_destination = false; - /* TODO: check if this is a real rare destination */ if(f->getFlowServerInfo() != NULL) { - u_int32_t hash = 0; + + u_int32_t hash; if(!getDestinationHash(f,&hash)) { return; } /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare destination hash %u", hash); */ @@ -72,40 +73,47 @@ void RareDestination::protocolDetected(Flow *f) { ndpi_bitmap *rare_dest_revise = cli_host->getRareDestReviseBMap(); time_t t_now = time(NULL); + + /* check if training has to start */ if (!ndpi_bitmap_cardinality(rare_dest)) { cli_host->clearSeenRareDestTraining(); - cli_host->setOngoingRareDestTraining(true); cli_host->setStartRareDestTraining(t_now); } - if ( cli_host->isOngoingRareDestTraining() - && cli_host->getSeenRareDestTraining() >= RARE_DEST_FLOWS_TO_SEE_TRAINING - && t_now - cli_host->getStartRareDestTraining() >= RARE_DEST_DURATION_TRAINING ) - cli_host->setOngoingRareDestTraining(false); - - if (!cli_host->isOngoingRareDestTraining() && t_now - cli_host->getRareDestLastEpoch() >= RARE_DEST_EPOCH_DURATION) { - if (t_now - cli_host->getRareDestLastEpoch() >= 2*RARE_DEST_EPOCH_DURATION) { - ndpi_bitmap_clear(rare_dest); - ndpi_bitmap_clear(rare_dest_revise); - } else { - ndpi_bitmap_xor(rare_dest_revise, rare_dest); // updates rare_dest_revise - ndpi_bitmap_and(rare_dest, rare_dest_revise); // makes rare_dest = rare_dest_revise + /* if host is training */ + if (cli_host->getStartRareDestTraining()) { + /* update */ + if (!ndpi_bitmap_isset(rare_dest, hash)) { + ndpi_bitmap_set(rare_dest, hash); + cli_host->incrementSeenRareDestTraining(); } - cli_host->setRareDestLastEpoch(t_now); + /* check if training has to end */ + if ( cli_host->getSeenRareDestTraining() >= RARE_DEST_FLOWS_TO_SEE_TRAINING + && t_now - cli_host->getStartRareDestTraining() >= RARE_DEST_DURATION_TRAINING ) + cli_host->setStartRareDestTraining(0); } - if (!hash) { + else { + /* check epoch */ + if (t_now - cli_host->getRareDestLastEpoch() >= RARE_DEST_EPOCH_DURATION) { + if (t_now - cli_host->getRareDestLastEpoch() >= 2*RARE_DEST_EPOCH_DURATION) { + ndpi_bitmap_clear(rare_dest); + ndpi_bitmap_clear(rare_dest_revise); + } + else { + ndpi_bitmap_xor(rare_dest_revise, rare_dest); // updates rare_dest_revise + ndpi_bitmap_and(rare_dest, rare_dest_revise); // makes rare_dest = rare_dest_revise + } + cli_host->setRareDestLastEpoch(t_now); + } + /* update */ if (ndpi_bitmap_isset(rare_dest, hash)) ndpi_bitmap_unset(rare_dest_revise, hash); else { ndpi_bitmap_set(rare_dest, hash); - if (!cli_host->isOngoingRareDestTraining()) is_rare_destination = true; - else cli_host->incrementSeenRareDestTraining(); + is_rare_destination = true; } } - /* error: hashing not possible */ - - //is_rare_destination = true; } if(is_rare_destination) { From bb399cf8f08dfff44bbf8c7bee02c2afaca8858a Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Wed, 10 May 2023 15:52:22 +0200 Subject: [PATCH 17/50] refactor & fixes: refactored getDestinationHash serialization of training variables into redis refactor epoch feature code --- include/Host.h | 13 ------ src/Host.cpp | 70 ++++++++++++++++++++--------- src/flow_checks/RareDestination.cpp | 66 ++++++++++++++------------- 3 files changed, 84 insertions(+), 65 deletions(-) diff --git a/include/Host.h b/include/Host.h index 92e581585fdb..ee1bf69ff355 100644 --- a/include/Host.h +++ b/include/Host.h @@ -74,13 +74,8 @@ class Host : public GenericHashEntry, public HostAlertableEntity, public Score, time_t last_epoch; struct { - bool ongoing; time_t start; u_int32_t seen; - /* - time_t duration = RARE_DEST_DURATION_TRAINING; - u_int32_t toSee = RARE_DEST_FLOWS_TO_SEE_TRAINING; - */ } rareDestTraining; /* END Host data: */ @@ -717,9 +712,6 @@ class Host : public GenericHashEntry, public HostAlertableEntity, public Score, inline time_t getRareDestLastEpoch() const { return(last_epoch); } inline void setRareDestLastEpoch(time_t t) { last_epoch=t; } - - inline bool isOngoingRareDestTraining() const { return(rareDestTraining.ongoing); } - inline void setOngoingRareDestTraining(bool b) { rareDestTraining.ongoing = b; } inline time_t getStartRareDestTraining() const { return(rareDestTraining.start); } inline void setStartRareDestTraining(time_t t) { rareDestTraining.start = t; } @@ -727,11 +719,6 @@ class Host : public GenericHashEntry, public HostAlertableEntity, public Score, inline u_int32_t getSeenRareDestTraining() const { return(rareDestTraining.seen); } inline void clearSeenRareDestTraining() { rareDestTraining.seen = 0; } inline void incrementSeenRareDestTraining() { rareDestTraining.seen++; } - - /* - inline time_t getDurationRareDestTraining() const { return(rareDestTraining.duration); } - inline u_int32_t getToSeeRareDestTraining() const { return(rareDestTraining.toSee); } - */ void dumpRareDestToRedis(); bool loadRareDestFromRedis(); diff --git a/src/Host.cpp b/src/Host.cpp index 6753747ce4e0..9ff2eae437da 100644 --- a/src/Host.cpp +++ b/src/Host.cpp @@ -308,9 +308,15 @@ void Host::initialize(Mac *_mac, u_int16_t _vlanId, u_int16_t observation_point_ ndpi_hll_init(&incoming_hosts_tcp_udp_port_with_no_tx_hll, 5 /* StdError: 18.4% */); if( !loadRareDestFromRedis() ){ - last_epoch = 0; - rare_dest = ndpi_bitmap_alloc(); - rare_dest_revise = ndpi_bitmap_alloc(); + if (!rare_dest) + rare_dest = ndpi_bitmap_alloc(); + else + ndpi_bitmap_clear(rare_dest); + + if (!rare_dest_revise) + rare_dest_revise = ndpi_bitmap_alloc(); + else + ndpi_bitmap_clear(rare_dest_revise); } deferredInitialization(); /* TODO To be called asynchronously for improving performance */ @@ -2598,7 +2604,7 @@ void Host::resetHostContacts() { /* *************************************** */ void Host::dumpRareDestToRedis() { - char key[CONST_MAX_LEN_REDIS_KEY], buf[32], last_epoch_ser[32], size_ser[32], *value; + char key[CONST_MAX_LEN_REDIS_KEY], buf[32], param_ser[32], *value; Redis *redis = ntop->getRedis(); size_t size; @@ -2617,8 +2623,8 @@ void Host::dumpRareDestToRedis() { } snprintf(buf, sizeof(buf), "rare_dest_len"); - snprintf(size_ser, sizeof(size_ser), "%lu", size); - redis->hashSet(key, buf, size_ser); + snprintf(param_ser, sizeof(param_ser), "%zu", size); + redis->hashSet(key, buf, param_ser); snprintf(buf, sizeof(buf), "rare_dest_revise"); size = ndpi_bitmap_serialize(rare_dest_revise, &value); @@ -2632,19 +2638,27 @@ void Host::dumpRareDestToRedis() { } snprintf(buf, sizeof(buf), "rare_dest_revise_len"); - snprintf(size_ser, sizeof(size_ser), "%lu", size); - redis->hashSet(key, buf, size_ser); + snprintf(param_ser, sizeof(param_ser), "%zu", size); + redis->hashSet(key, buf, param_ser); snprintf(buf, sizeof(buf), "last_epoch"); - snprintf(last_epoch_ser, sizeof(last_epoch_ser), "%ld", last_epoch); + snprintf(param_ser, sizeof(param_ser), "%ld", last_epoch); + redis->hashSet(key, buf, param_ser); + + snprintf(buf, sizeof(buf), "t_start_training"); + snprintf(param_ser, sizeof(param_ser), "%ld", rareDestTraining.start); + redis->hashSet(key, buf, param_ser); - redis->hashSet(key, buf, last_epoch_ser); - ntop->getTrace()->traceEvent(TRACE_NORMAL, "Dump to redis finished"); + snprintf(buf, sizeof(buf), "seen_flows_training"); + snprintf(param_ser, sizeof(param_ser), "%lu", (unsigned long)rareDestTraining.seen); + redis->hashSet(key, buf, param_ser); + + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data dumped to redis"); } bool Host::loadRareDestFromRedis() { - char key[CONST_MAX_LEN_REDIS_KEY], buf[32], last_epoch_str[32], size_str[32], *value; + char key[CONST_MAX_LEN_REDIS_KEY], buf[32], param_str[32], *value; Redis *redis = ntop->getRedis(); size_t size; @@ -2653,12 +2667,20 @@ bool Host::loadRareDestFromRedis() { snprintf(key, sizeof(key), HOST_RARE_DEST_SERIALIZED_KEY, iface->get_id()); snprintf(buf, sizeof(buf), "last_epoch"); - if( redis->hashGet(key, buf, last_epoch_str, sizeof(last_epoch_str)) != 0 ) return(false); - last_epoch = atol(last_epoch_str); + if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); + last_epoch = atol(param_str); + + snprintf(buf, sizeof(buf), "t_start_training"); + if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); + rareDestTraining.start = atol(param_str); + + snprintf(buf, sizeof(buf), "seen_flows_training"); + if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); + rareDestTraining.seen = (u_int32_t)strtoul(param_str, NULL, 10); snprintf(buf, sizeof(buf), "rare_dest_len"); - if( redis->hashGet(key, buf, size_str, sizeof(size_str)) != 0 ) return(false); - size = (size_t)strtoul(size_str, NULL, 10); + if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); + size = (size_t)strtoul(param_str, NULL, 10); if((value = (char *) malloc(size)) == NULL) { ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to allocate memory to deserialize %s", key); @@ -2666,14 +2688,16 @@ bool Host::loadRareDestFromRedis() { } snprintf(buf, sizeof(buf), "rare_dest"); - if(redis->hashGet(key, buf, value, size) == 0) { - rare_dest = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); + if(redis->hashGet(key, buf, value, size) != 0) { + free(value); + return(false); } + rare_dest = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); free(value); snprintf(buf, sizeof(buf), "rare_dest_revise_len"); - if( redis->hashGet(key, buf, size_str, sizeof(size_str)) != 0 ) return(false); - size = (size_t)strtoul(size_str, NULL, 10); + if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); + size = (size_t)strtoul(param_str, NULL, 10); if((value = (char *) malloc(size)) == NULL) { ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to allocate memory to deserialize %s", key); @@ -2681,9 +2705,11 @@ bool Host::loadRareDestFromRedis() { } snprintf(buf, sizeof(buf), "rare_dest_revise"); - if(redis->hashGet(key, buf, value, size) == 0) { - rare_dest_revise = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); + if(redis->hashGet(key, buf, value, size) != 0) { + free(value); + return(false); } + rare_dest_revise = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); free(value); return(true); diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index 933758af9e0f..fc3a53417364 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -24,32 +24,30 @@ /* ***************************************************** */ -bool getDestinationHash(Flow *f, u_int32_t *hash) { +u_int32_t getDestinationHash(Flow *f) { + u_int32_t hash = 0; Host *dest = f->get_srv_host(); if (f->isLocalToLocal()) { - char buf[128]; + char buf[64]; if (!dest->isMulticastHost() && dest->isDHCPHost()) { char *mac = dest->getMac()->print(buf,sizeof(buf)); - *hash = Utils::hashString(mac); + hash = Utils::hashString(mac); /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination MAC %u - %s", *hash, mac); */ - return (true); } if (dest->isIPv6() || dest->isIPv4()) { char *ip = dest->get_ip()->print(buf,sizeof(buf)); - *hash = Utils::hashString(ip); + hash = Utils::hashString(ip); /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination IPv6/IPv4 %u - %s", *hash, ip); */ - return (true); } } if (f->isLocalToRemote()) { char name_buf[128]; char *domain = dest->get_name(name_buf, sizeof(name_buf), false); - *hash = Utils::hashString(domain); + hash = Utils::hashString(domain); /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare remote destination %u - %s", *hash, domain); */ - return (true); } - return (false); + return hash; } /* ***************************************************** */ @@ -64,14 +62,16 @@ void RareDestination::protocolDetected(Flow *f) { if(f->getFlowServerInfo() != NULL) { - u_int32_t hash; - if(!getDestinationHash(f,&hash)) { return; } + u_int32_t hash = getDestinationHash(f); + if(hash == 0) { return; } /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare destination hash %u", hash); */ Host *cli_host = f->get_cli_host(); ndpi_bitmap *rare_dest = cli_host->getRareDestBMap(); ndpi_bitmap *rare_dest_revise = cli_host->getRareDestReviseBMap(); + /* TODO: Check if bitmap pointers are valid */ + time_t t_now = time(NULL); /* check if training has to start */ @@ -90,30 +90,36 @@ void RareDestination::protocolDetected(Flow *f) { /* check if training has to end */ if ( cli_host->getSeenRareDestTraining() >= RARE_DEST_FLOWS_TO_SEE_TRAINING && t_now - cli_host->getStartRareDestTraining() >= RARE_DEST_DURATION_TRAINING ) + { cli_host->setStartRareDestTraining(0); + cli_host->setRareDestLastEpoch(t_now); + } + + return; } + /* check epoch */ + + if (t_now - cli_host->getRareDestLastEpoch() >= 2*RARE_DEST_EPOCH_DURATION) { + ndpi_bitmap_clear(rare_dest); + ndpi_bitmap_clear(rare_dest_revise); + return; + } + + if (t_now - cli_host->getRareDestLastEpoch() >= RARE_DEST_EPOCH_DURATION) { + ndpi_bitmap_xor(rare_dest_revise, rare_dest); // updates rare_dest_revise + ndpi_bitmap_and(rare_dest, rare_dest_revise); // makes rare_dest = rare_dest_revise + cli_host->setRareDestLastEpoch(t_now); + } + + /* update */ + if (ndpi_bitmap_isset(rare_dest, hash)) + ndpi_bitmap_unset(rare_dest_revise, hash); else { - /* check epoch */ - if (t_now - cli_host->getRareDestLastEpoch() >= RARE_DEST_EPOCH_DURATION) { - if (t_now - cli_host->getRareDestLastEpoch() >= 2*RARE_DEST_EPOCH_DURATION) { - ndpi_bitmap_clear(rare_dest); - ndpi_bitmap_clear(rare_dest_revise); - } - else { - ndpi_bitmap_xor(rare_dest_revise, rare_dest); // updates rare_dest_revise - ndpi_bitmap_and(rare_dest, rare_dest_revise); // makes rare_dest = rare_dest_revise - } - cli_host->setRareDestLastEpoch(t_now); - } - /* update */ - if (ndpi_bitmap_isset(rare_dest, hash)) - ndpi_bitmap_unset(rare_dest_revise, hash); - else { - ndpi_bitmap_set(rare_dest, hash); - is_rare_destination = true; - } + ndpi_bitmap_set(rare_dest, hash); + is_rare_destination = true; } + } if(is_rare_destination) { From a62d3dcdca16feee37bab5b33b1991ace16336a5 Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Fri, 12 May 2023 15:39:23 +0200 Subject: [PATCH 18/50] Testing feature & fixes --- include/ntop_defines.h | 4 ++-- src/Host.cpp | 33 ++++++++++++++++------------- src/flow_checks/RareDestination.cpp | 11 +++++++--- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/include/ntop_defines.h b/include/ntop_defines.h index de54386e72b9..8f990938273d 100644 --- a/include/ntop_defines.h +++ b/include/ntop_defines.h @@ -1307,8 +1307,8 @@ extern struct ntopngLuaContext* getUserdata(struct lua_State *vm); /******************************************************************************/ #define HOST_RARE_DEST_SERIALIZED_KEY "ntopng.serialized_host_rare_dest_fields.ifid_%u" -#define RARE_DEST_DURATION_TRAINING 3600 /* seconds */ -#define RARE_DEST_FLOWS_TO_SEE_TRAINING 100 +#define RARE_DEST_DURATION_TRAINING 60 /* seconds */ +#define RARE_DEST_FLOWS_TO_SEE_TRAINING 10 #define RARE_DEST_EPOCH_DURATION 3600 /* seconds */ /******************************************************************************/ diff --git a/src/Host.cpp b/src/Host.cpp index 9ff2eae437da..45355213e87a 100644 --- a/src/Host.cpp +++ b/src/Host.cpp @@ -108,7 +108,7 @@ Host::~Host() { if(externalAlert.msg) free(externalAlert.msg); if(tcp_udp_contacted_ports_no_tx) ndpi_bitmap_free(tcp_udp_contacted_ports_no_tx); - dumpRareDestToRedis(); + //dumpRareDestToRedis(); if(rare_dest) ndpi_bitmap_free(rare_dest); if(rare_dest_revise) ndpi_bitmap_free(rare_dest_revise); @@ -308,15 +308,12 @@ void Host::initialize(Mac *_mac, u_int16_t _vlanId, u_int16_t observation_point_ ndpi_hll_init(&incoming_hosts_tcp_udp_port_with_no_tx_hll, 5 /* StdError: 18.4% */); if( !loadRareDestFromRedis() ){ - if (!rare_dest) - rare_dest = ndpi_bitmap_alloc(); - else - ndpi_bitmap_clear(rare_dest); - - if (!rare_dest_revise) - rare_dest_revise = ndpi_bitmap_alloc(); - else - ndpi_bitmap_clear(rare_dest_revise); + rare_dest = ndpi_bitmap_alloc(); + rare_dest_revise = ndpi_bitmap_alloc(); + last_epoch = time(NULL); + setStartRareDestTraining(time(NULL)); + clearSeenRareDestTraining(); + dumpRareDestToRedis(); } deferredInitialization(); /* TODO To be called asynchronously for improving performance */ @@ -2616,14 +2613,14 @@ void Host::dumpRareDestToRedis() { size = ndpi_bitmap_serialize(rare_dest, &value); if (value) { char *encoded_bmap = Utils::base64_encode((unsigned char *)value, size); - size = strlen(encoded_bmap)+2; + size = strlen(encoded_bmap); redis->hashSet(key, buf, encoded_bmap); free(encoded_bmap); free(value); } snprintf(buf, sizeof(buf), "rare_dest_len"); - snprintf(param_ser, sizeof(param_ser), "%zu", size); + snprintf(param_ser, sizeof(param_ser), "%lu", size); redis->hashSet(key, buf, param_ser); snprintf(buf, sizeof(buf), "rare_dest_revise"); @@ -2631,14 +2628,14 @@ void Host::dumpRareDestToRedis() { if (value) { char *encoded_bmap = Utils::base64_encode((unsigned char *)value, size); - size = strlen(encoded_bmap)+2; + size = strlen(encoded_bmap); redis->hashSet(key, buf, encoded_bmap); free(encoded_bmap); free(value); } snprintf(buf, sizeof(buf), "rare_dest_revise_len"); - snprintf(param_ser, sizeof(param_ser), "%zu", size); + snprintf(param_ser, sizeof(param_ser), "%lu", size); redis->hashSet(key, buf, param_ser); @@ -2654,7 +2651,7 @@ void Host::dumpRareDestToRedis() { snprintf(param_ser, sizeof(param_ser), "%lu", (unsigned long)rareDestTraining.seen); redis->hashSet(key, buf, param_ser); - ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data dumped to redis"); + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data dumped to redis last_epoch: %ld RareDestTraining.start: %ld rareDestTraining.seen: %lu", last_epoch, rareDestTraining.start, (unsigned long)rareDestTraining.seen); } bool Host::loadRareDestFromRedis() { @@ -2692,6 +2689,10 @@ bool Host::loadRareDestFromRedis() { free(value); return(false); } + + /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "rare_dest size %lu", size); + ntop->getTrace()->traceEvent(TRACE_NORMAL, "rare_dest encoded: %s", value ); */ + rare_dest = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); free(value); @@ -2712,5 +2713,7 @@ bool Host::loadRareDestFromRedis() { rare_dest_revise = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); free(value); + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data loaded from redis , last_epoch: %ld RareDestTraining.start: %ld RareDestTraining.seen: %lu", last_epoch, rareDestTraining.start, rareDestTraining.seen); + return(true); } diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index fc3a53417364..4d61a09a1528 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -64,7 +64,6 @@ void RareDestination::protocolDetected(Flow *f) { u_int32_t hash = getDestinationHash(f); if(hash == 0) { return; } - /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare destination hash %u", hash); */ Host *cli_host = f->get_cli_host(); ndpi_bitmap *rare_dest = cli_host->getRareDestBMap(); @@ -78,6 +77,7 @@ void RareDestination::protocolDetected(Flow *f) { if (!ndpi_bitmap_cardinality(rare_dest)) { cli_host->clearSeenRareDestTraining(); cli_host->setStartRareDestTraining(t_now); + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training On at %ld", t_now); } /* if host is training */ @@ -86,6 +86,7 @@ void RareDestination::protocolDetected(Flow *f) { if (!ndpi_bitmap_isset(rare_dest, hash)) { ndpi_bitmap_set(rare_dest, hash); cli_host->incrementSeenRareDestTraining(); + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Hash added for %s", f->getFlowServerInfo()); } /* check if training has to end */ if ( cli_host->getSeenRareDestTraining() >= RARE_DEST_FLOWS_TO_SEE_TRAINING @@ -93,8 +94,9 @@ void RareDestination::protocolDetected(Flow *f) { { cli_host->setStartRareDestTraining(0); cli_host->setRareDestLastEpoch(t_now); + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training Off at %ld", t_now); } - + cli_host->dumpRareDestToRedis(); return; } @@ -103,11 +105,12 @@ void RareDestination::protocolDetected(Flow *f) { if (t_now - cli_host->getRareDestLastEpoch() >= 2*RARE_DEST_EPOCH_DURATION) { ndpi_bitmap_clear(rare_dest); ndpi_bitmap_clear(rare_dest_revise); + cli_host->dumpRareDestToRedis(); return; } if (t_now - cli_host->getRareDestLastEpoch() >= RARE_DEST_EPOCH_DURATION) { - ndpi_bitmap_xor(rare_dest_revise, rare_dest); // updates rare_dest_revise + //ndpi_bitmap_xor(rare_dest_revise, rare_dest); // updates rare_dest_revise ndpi_bitmap_and(rare_dest, rare_dest_revise); // makes rare_dest = rare_dest_revise cli_host->setRareDestLastEpoch(t_now); } @@ -120,6 +123,8 @@ void RareDestination::protocolDetected(Flow *f) { is_rare_destination = true; } + cli_host->dumpRareDestToRedis(); + } if(is_rare_destination) { From 7f21dbc3758f884113cd1c523efc9362a29ce2d2 Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Sat, 13 May 2023 01:15:04 +0200 Subject: [PATCH 19/50] fix: fixed getDestinationHash's function signature --- include/flow_checks/RareDestination.h | 3 +++ src/flow_checks/RareDestination.cpp | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/include/flow_checks/RareDestination.h b/include/flow_checks/RareDestination.h index 408ced69466c..b667ee8bde32 100644 --- a/include/flow_checks/RareDestination.h +++ b/include/flow_checks/RareDestination.h @@ -26,6 +26,9 @@ class RareDestination : public FlowCheck { + private: + u_int32_t getDestinationHash(Flow *f); + public: RareDestination() : FlowCheck(ntopng_edition_community, true /* Packet Interfaces only */, true /* Exclude for nEdge */, false /* Only for nEdge */, diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index 4d61a09a1528..6a07b6e33dad 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -24,7 +24,7 @@ /* ***************************************************** */ -u_int32_t getDestinationHash(Flow *f) { +u_int32_t RareDestination::getDestinationHash(Flow *f) { u_int32_t hash = 0; Host *dest = f->get_srv_host(); if (f->isLocalToLocal()) { @@ -45,7 +45,7 @@ u_int32_t getDestinationHash(Flow *f) { char name_buf[128]; char *domain = dest->get_name(name_buf, sizeof(name_buf), false); hash = Utils::hashString(domain); - /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare remote destination %u - %s", *hash, domain); */ + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare remote destination %u - %s - %s", hash, domain, dest->getServerName(name_buf, sizeof(name_buf))); } return hash; } From cdda89736f015d0d7ba92cfe9ff9d626e633753a Mon Sep 17 00:00:00 2001 From: Leonardo Brugnano Date: Sat, 13 May 2023 11:35:57 +0200 Subject: [PATCH 20/50] [fix] loadRareDestFromRedis memory leak --- src/Host.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Host.cpp b/src/Host.cpp index 45355213e87a..201b4c8cf3fa 100644 --- a/src/Host.cpp +++ b/src/Host.cpp @@ -2697,17 +2697,22 @@ bool Host::loadRareDestFromRedis() { free(value); snprintf(buf, sizeof(buf), "rare_dest_revise_len"); - if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); + if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) { + ndpi_bitmap_free(rare_dest); + return(false); + } size = (size_t)strtoul(param_str, NULL, 10); if((value = (char *) malloc(size)) == NULL) { ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to allocate memory to deserialize %s", key); + ndpi_bitmap_free(rare_dest); return(false); } snprintf(buf, sizeof(buf), "rare_dest_revise"); if(redis->hashGet(key, buf, value, size) != 0) { free(value); + ndpi_bitmap_free(rare_dest); return(false); } rare_dest_revise = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); From 09231d23b8e36e37181c9b0249d657c3149b7810 Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Thu, 18 May 2023 23:01:31 +0200 Subject: [PATCH 21/50] refactor: moved implementation into LocalHost --- include/Host.h | 19 +--- include/LocalHost.h | 45 ++++++--- include/ntop_defines.h | 2 +- src/Host.cpp | 131 +------------------------ src/LocalHost.cpp | 145 +++++++++++++++++++++++++++- src/flow_checks/RareDestination.cpp | 56 +++++------ 6 files changed, 207 insertions(+), 191 deletions(-) diff --git a/include/Host.h b/include/Host.h index e91d5442e1b6..b30ee335a090 100644 --- a/include/Host.h +++ b/include/Host.h @@ -75,7 +75,7 @@ class Host : public GenericHashEntry, } names; char *ssdpLocation; - + /* END Host data: */ /* Counters used by host alerts */ @@ -1000,24 +1000,7 @@ class Host : public GenericHashEntry, ndpi_bitmap_set(tcp_udp_contacted_ports_no_tx, port); } - /* RareDest Extension methods */ - - inline ndpi_bitmap* getRareDestBMap() const { return(rare_dest); } - inline ndpi_bitmap* getRareDestReviseBMap() const { return(rare_dest_revise); } - - inline time_t getRareDestLastEpoch() const { return(last_epoch); } - inline void setRareDestLastEpoch(time_t t) { last_epoch=t; } - inline time_t getStartRareDestTraining() const { return(rareDestTraining.start); } - inline void setStartRareDestTraining(time_t t) { rareDestTraining.start = t; } - - inline u_int32_t getSeenRareDestTraining() const { return(rareDestTraining.seen); } - inline void clearSeenRareDestTraining() { rareDestTraining.seen = 0; } - inline void incrementSeenRareDestTraining() { rareDestTraining.seen++; } - - void dumpRareDestToRedis(); - bool loadRareDestFromRedis(); - void resetHostContacts(); }; diff --git a/include/LocalHost.h b/include/LocalHost.h index be9dd5326964..839236c04086 100644 --- a/include/LocalHost.h +++ b/include/LocalHost.h @@ -36,6 +36,16 @@ class LocalHost : public Host { u_int8_t router_mac_set : 1, drop_all_host_traffic : 1, systemHost : 1, _notused : 5; + /* RareDestination data implementation*/ + ndpi_bitmap *rare_dest; + ndpi_bitmap *rare_dest_revise; + time_t last_epoch; + + struct { + time_t start; + u_int32_t seen; + } rareDestTraining; + /* LocalHost data: update LocalHost::deleteHostData when adding new fields */ char *os_detail; /* END Host data: */ @@ -177,18 +187,29 @@ class LocalHost : public Host { void setRouterMac(Mac *gw); - inline void setServerPort(bool isTCP, u_int16_t port, ndpi_protocol *proto) { - usedPorts.setServerPort(isTCP, port, proto); - }; - inline void setContactedPort(bool isTCP, u_int16_t port, - ndpi_protocol *proto) { - usedPorts.setContactedPort(isTCP, port, proto); - }; - virtual inline void luaUsedPorts(lua_State *vm) { usedPorts.lua(vm, iface); }; - virtual inline std::unordered_map *getServerPorts( - bool isTCP) { - return (usedPorts.getServerPorts(isTCP)); - }; + inline void setServerPort(bool isTCP, u_int16_t port, ndpi_protocol *proto) { usedPorts.setServerPort(isTCP, port, proto); }; + inline void setContactedPort(bool isTCP, u_int16_t port, ndpi_protocol *proto) { usedPorts.setContactedPort(isTCP, port, proto); }; + virtual inline void luaUsedPorts(lua_State* vm) { usedPorts.lua(vm, iface); }; + virtual inline std::unordered_map* getServerPorts(bool isTCP) { return(usedPorts.getServerPorts(isTCP)); }; + + /* RareDest Extension methods */ + + inline ndpi_bitmap* getRareDestBMap() const { return(rare_dest); } + inline ndpi_bitmap* getRareDestReviseBMap() const { return(rare_dest_revise); } + + inline time_t getRareDestLastEpoch() const { return(last_epoch); } + inline void setRareDestLastEpoch(time_t t) { last_epoch=t; } + + inline time_t getStartRareDestTraining() const { return(rareDestTraining.start); } + inline void setStartRareDestTraining(time_t t) { rareDestTraining.start = t; } + + inline u_int32_t getSeenRareDestTraining() const { return(rareDestTraining.seen); } + inline void clearSeenRareDestTraining() { rareDestTraining.seen = 0; } + inline void incrementSeenRareDestTraining() { rareDestTraining.seen++; } + + void dumpRareDestToRedis(); + bool loadRareDestFromRedis(); + }; #endif /* _LOCAL_HOST_H_ */ diff --git a/include/ntop_defines.h b/include/ntop_defines.h index 900032001471..729f445e8fe0 100644 --- a/include/ntop_defines.h +++ b/include/ntop_defines.h @@ -1487,7 +1487,7 @@ extern struct ntopngLuaContext *getUserdata(struct lua_State *vm); /******************************************************************************/ -#define HOST_RARE_DEST_SERIALIZED_KEY "ntopng.serialized_host_rare_dest_fields.ifid_%u" +#define HOST_RARE_DEST_SERIALIZED_KEY "ntopng.localhost_rare_dest_fields.%s" #define RARE_DEST_DURATION_TRAINING 60 /* seconds */ #define RARE_DEST_FLOWS_TO_SEE_TRAINING 10 #define RARE_DEST_EPOCH_DURATION 3600 /* seconds */ diff --git a/src/Host.cpp b/src/Host.cpp index 20f47cfc38a9..cca7e1f8cf0b 100644 --- a/src/Host.cpp +++ b/src/Host.cpp @@ -117,10 +117,6 @@ Host::~Host() { if (tcp_udp_contacted_ports_no_tx) ndpi_bitmap_free(tcp_udp_contacted_ports_no_tx); - //dumpRareDestToRedis(); - if(rare_dest) ndpi_bitmap_free(rare_dest); - if(rare_dest_revise) ndpi_bitmap_free(rare_dest_revise); - ndpi_hll_destroy(&outgoing_hosts_tcp_udp_port_with_no_tx_hll); ndpi_hll_destroy(&incoming_hosts_tcp_udp_port_with_no_tx_hll); } @@ -336,7 +332,7 @@ void Host::initialize(Mac *_mac, u_int16_t _vlanId, ndpi_hll_init(&incoming_hosts_tcp_udp_port_with_no_tx_hll, 5 /* StdError: 18.4% */); - deferredInitialization(); /* TODO To be called asynchronously for improving + deferredInitialization(); /* TODO To be called asynchronously for improving performance */ } /* *************************************** */ @@ -2710,128 +2706,3 @@ void Host::resetHostContacts() { ndpi_hll_reset(&incoming_hosts_tcp_udp_port_with_no_tx_hll); ndpi_bitmap_clear(tcp_udp_contacted_ports_no_tx); } - -/* *************************************** */ - -void Host::dumpRareDestToRedis() { - char key[CONST_MAX_LEN_REDIS_KEY], buf[32], param_ser[32], *value; - Redis *redis = ntop->getRedis(); - size_t size; - - if((!redis) || (!iface) || (!rare_dest) || (!rare_dest_revise)) return; - - snprintf(key, sizeof(key), HOST_RARE_DEST_SERIALIZED_KEY, iface->get_id()); - - snprintf(buf, sizeof(buf), "rare_dest"); - size = ndpi_bitmap_serialize(rare_dest, &value); - if (value) { - char *encoded_bmap = Utils::base64_encode((unsigned char *)value, size); - size = strlen(encoded_bmap); - redis->hashSet(key, buf, encoded_bmap); - free(encoded_bmap); - free(value); - } - - snprintf(buf, sizeof(buf), "rare_dest_len"); - snprintf(param_ser, sizeof(param_ser), "%lu", size); - redis->hashSet(key, buf, param_ser); - - snprintf(buf, sizeof(buf), "rare_dest_revise"); - size = ndpi_bitmap_serialize(rare_dest_revise, &value); - - if (value) { - char *encoded_bmap = Utils::base64_encode((unsigned char *)value, size); - size = strlen(encoded_bmap); - redis->hashSet(key, buf, encoded_bmap); - free(encoded_bmap); - free(value); - } - - snprintf(buf, sizeof(buf), "rare_dest_revise_len"); - snprintf(param_ser, sizeof(param_ser), "%lu", size); - redis->hashSet(key, buf, param_ser); - - - snprintf(buf, sizeof(buf), "last_epoch"); - snprintf(param_ser, sizeof(param_ser), "%ld", last_epoch); - redis->hashSet(key, buf, param_ser); - - snprintf(buf, sizeof(buf), "t_start_training"); - snprintf(param_ser, sizeof(param_ser), "%ld", rareDestTraining.start); - redis->hashSet(key, buf, param_ser); - - snprintf(buf, sizeof(buf), "seen_flows_training"); - snprintf(param_ser, sizeof(param_ser), "%lu", (unsigned long)rareDestTraining.seen); - redis->hashSet(key, buf, param_ser); - - //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data dumped to redis last_epoch: %ld RareDestTraining.start: %ld rareDestTraining.seen: %lu", last_epoch, rareDestTraining.start, (unsigned long)rareDestTraining.seen); -} - -bool Host::loadRareDestFromRedis() { - char key[CONST_MAX_LEN_REDIS_KEY], buf[32], param_str[32], *value; - Redis *redis = ntop->getRedis(); - size_t size; - - if((!redis) || (!iface)) return(false); - - snprintf(key, sizeof(key), HOST_RARE_DEST_SERIALIZED_KEY, iface->get_id()); - - snprintf(buf, sizeof(buf), "last_epoch"); - if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); - last_epoch = atol(param_str); - - snprintf(buf, sizeof(buf), "t_start_training"); - if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); - rareDestTraining.start = atol(param_str); - - snprintf(buf, sizeof(buf), "seen_flows_training"); - if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); - rareDestTraining.seen = (u_int32_t)strtoul(param_str, NULL, 10); - - snprintf(buf, sizeof(buf), "rare_dest_len"); - if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); - size = (size_t)strtoul(param_str, NULL, 10); - - if((value = (char *) malloc(size)) == NULL) { - ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to allocate memory to deserialize %s", key); - return(false); - } - - snprintf(buf, sizeof(buf), "rare_dest"); - if(redis->hashGet(key, buf, value, size) != 0) { - free(value); - return(false); - } - - /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "rare_dest size %lu", size); - ntop->getTrace()->traceEvent(TRACE_NORMAL, "rare_dest encoded: %s", value ); */ - - rare_dest = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); - free(value); - - snprintf(buf, sizeof(buf), "rare_dest_revise_len"); - if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) { - ndpi_bitmap_free(rare_dest); - return(false); - } - size = (size_t)strtoul(param_str, NULL, 10); - - if((value = (char *) malloc(size)) == NULL) { - ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to allocate memory to deserialize %s", key); - ndpi_bitmap_free(rare_dest); - return(false); - } - - snprintf(buf, sizeof(buf), "rare_dest_revise"); - if(redis->hashGet(key, buf, value, size) != 0) { - free(value); - ndpi_bitmap_free(rare_dest); - return(false); - } - rare_dest_revise = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); - free(value); - - //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data loaded from redis , last_epoch: %ld RareDestTraining.start: %ld RareDestTraining.seen: %lu", last_epoch, rareDestTraining.start, rareDestTraining.seen); - - return(true); -} diff --git a/src/LocalHost.cpp b/src/LocalHost.cpp index 657550207892..487c4eb9f1e3 100644 --- a/src/LocalHost.cpp +++ b/src/LocalHost.cpp @@ -45,8 +45,13 @@ LocalHost::LocalHost(NetworkInterface *_iface, char *ipAddress, /* *************************************** */ -LocalHost::~LocalHost() { - if (initial_ts_point) delete (initial_ts_point); +LocalHost::~LocalHost() { + if(initial_ts_point) delete(initial_ts_point); + + dumpRareDestToRedis(); + if(rare_dest) ndpi_bitmap_free(rare_dest); + if(rare_dest_revise) ndpi_bitmap_free(rare_dest_revise); + freeLocalHostData(); } @@ -141,6 +146,15 @@ void LocalHost::initialize() { #ifdef HAVE_NEDGE drop_all_host_traffic = 0; #endif + + if( !loadRareDestFromRedis() ){ + rare_dest = ndpi_bitmap_alloc(); + rare_dest_revise = ndpi_bitmap_alloc(); + last_epoch = time(NULL); + setStartRareDestTraining(time(NULL)); + clearSeenRareDestTraining(); + } + } /* *************************************** */ @@ -529,3 +543,130 @@ void LocalHost::setRxOnlyHost(bool set_it) { } } } + +/* *************************************** */ + +void LocalHost::dumpRareDestToRedis() { + char key[CONST_MAX_LEN_REDIS_KEY], buf[32], param_ser[32], *value; + Redis *redis = ntop->getRedis(); + size_t size; + + if((!redis) || (!rare_dest) || (!rare_dest_revise)) return; + + char hostbuf[64]; + snprintf(key, sizeof(key), HOST_RARE_DEST_SERIALIZED_KEY, get_hostkey(hostbuf, sizeof(hostbuf))); + + snprintf(buf, sizeof(buf), "rare_dest"); + size = ndpi_bitmap_serialize(rare_dest, &value); + if (value) { + char *encoded_bmap = Utils::base64_encode((unsigned char *)value, size); + size = strlen(encoded_bmap); + redis->hashSet(key, buf, encoded_bmap); + free(encoded_bmap); + free(value); + } + + snprintf(buf, sizeof(buf), "rare_dest_len"); + snprintf(param_ser, sizeof(param_ser), "%lu", size); + redis->hashSet(key, buf, param_ser); + + snprintf(buf, sizeof(buf), "rare_dest_revise"); + size = ndpi_bitmap_serialize(rare_dest_revise, &value); + + if (value) { + char *encoded_bmap = Utils::base64_encode((unsigned char *)value, size); + size = strlen(encoded_bmap); + redis->hashSet(key, buf, encoded_bmap); + free(encoded_bmap); + free(value); + } + + snprintf(buf, sizeof(buf), "rare_dest_revise_len"); + snprintf(param_ser, sizeof(param_ser), "%lu", size); + redis->hashSet(key, buf, param_ser); + + + snprintf(buf, sizeof(buf), "last_epoch"); + snprintf(param_ser, sizeof(param_ser), "%ld", last_epoch); + redis->hashSet(key, buf, param_ser); + + snprintf(buf, sizeof(buf), "t_start_training"); + snprintf(param_ser, sizeof(param_ser), "%ld", rareDestTraining.start); + redis->hashSet(key, buf, param_ser); + + snprintf(buf, sizeof(buf), "seen_flows_training"); + snprintf(param_ser, sizeof(param_ser), "%lu", (unsigned long)rareDestTraining.seen); + redis->hashSet(key, buf, param_ser); + + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data dumped to redis last_epoch: %ld RareDestTraining.start: %ld rareDestTraining.seen: %lu", last_epoch, rareDestTraining.start, (unsigned long)rareDestTraining.seen); +} + +bool LocalHost::loadRareDestFromRedis() { + char key[CONST_MAX_LEN_REDIS_KEY], buf[32], param_str[32], *value; + Redis *redis = ntop->getRedis(); + size_t size; + + if((!redis)) return(false); + + char hostbuf[64]; + snprintf(key, sizeof(key), HOST_RARE_DEST_SERIALIZED_KEY, get_hostkey(hostbuf, sizeof(hostbuf))); + + snprintf(buf, sizeof(buf), "last_epoch"); + if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); + last_epoch = atol(param_str); + + snprintf(buf, sizeof(buf), "t_start_training"); + if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); + rareDestTraining.start = atol(param_str); + + snprintf(buf, sizeof(buf), "seen_flows_training"); + if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); + rareDestTraining.seen = (u_int32_t)strtoul(param_str, NULL, 10); + + snprintf(buf, sizeof(buf), "rare_dest_len"); + if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); + size = (size_t)strtoul(param_str, NULL, 10); + + if((value = (char *) malloc(size)) == NULL) { + ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to allocate memory to deserialize %s", key); + return(false); + } + + snprintf(buf, sizeof(buf), "rare_dest"); + if(redis->hashGet(key, buf, value, size) != 0) { + free(value); + return(false); + } + + /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "rare_dest size %lu", size); + ntop->getTrace()->traceEvent(TRACE_NORMAL, "rare_dest encoded: %s", value ); */ + + rare_dest = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); + free(value); + + snprintf(buf, sizeof(buf), "rare_dest_revise_len"); + if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) { + ndpi_bitmap_free(rare_dest); + return(false); + } + size = (size_t)strtoul(param_str, NULL, 10); + + if((value = (char *) malloc(size)) == NULL) { + ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to allocate memory to deserialize %s", key); + ndpi_bitmap_free(rare_dest); + return(false); + } + + snprintf(buf, sizeof(buf), "rare_dest_revise"); + if(redis->hashGet(key, buf, value, size) != 0) { + free(value); + ndpi_bitmap_free(rare_dest); + return(false); + } + rare_dest_revise = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); + free(value); + + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data loaded from redis , last_epoch: %ld RareDestTraining.start: %ld RareDestTraining.seen: %lu", last_epoch, rareDestTraining.start, rareDestTraining.seen); + + return(true); +} diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index c9708ee46b15..f95331d07967 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -52,67 +52,66 @@ u_int32_t RareDestination::getDestinationHash(Flow *f) { /* ***************************************************** */ -/* - ndpi_bitmap_xor(ndpi_bitmap*, ndpi_bitmap*) defined in ndpi_api.h, ndpi_bitmap.c -*/ - void RareDestination::protocolDetected(Flow *f) { bool is_rare_destination = false; if(f->getFlowServerInfo() != NULL) { - + if(!f->get_cli_host()->isLocalHost()) return; + u_int32_t hash = getDestinationHash(f); - if(hash == 0) { return; } - - Host *cli_host = f->get_cli_host(); - ndpi_bitmap *rare_dest = cli_host->getRareDestBMap(); - ndpi_bitmap *rare_dest_revise = cli_host->getRareDestReviseBMap(); + if(hash == 0) return; + + LocalHost *cli_lhost = (LocalHost*)f->get_cli_host(); + ndpi_bitmap *rare_dest = cli_lhost->getRareDestBMap(); + ndpi_bitmap *rare_dest_revise = cli_lhost->getRareDestReviseBMap(); + char hostbuf[64], *host_id; + host_id = cli_lhost->get_hostkey(hostbuf, sizeof(hostbuf)); /* TODO: Check if bitmap pointers are valid */ time_t t_now = time(NULL); /* check if training has to start */ if (!ndpi_bitmap_cardinality(rare_dest)) { - cli_host->clearSeenRareDestTraining(); - cli_host->setStartRareDestTraining(t_now); - ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training On at %ld", t_now); + cli_lhost->clearSeenRareDestTraining(); + cli_lhost->setStartRareDestTraining(t_now); + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training On at %ld ~ %s", t_now, host_id ); } /* if host is training */ - if (cli_host->getStartRareDestTraining()) { + if (cli_lhost->getStartRareDestTraining()) { /* update */ if (!ndpi_bitmap_isset(rare_dest, hash)) { ndpi_bitmap_set(rare_dest, hash); - cli_host->incrementSeenRareDestTraining(); - ntop->getTrace()->traceEvent(TRACE_NORMAL, "Hash added for %s", f->getFlowServerInfo()); + cli_lhost->incrementSeenRareDestTraining(); + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Hash %s added ~ %s", f->getFlowServerInfo(), host_id ); } /* check if training has to end */ - if ( cli_host->getSeenRareDestTraining() >= RARE_DEST_FLOWS_TO_SEE_TRAINING - && t_now - cli_host->getStartRareDestTraining() >= RARE_DEST_DURATION_TRAINING ) + if ( cli_lhost->getSeenRareDestTraining() >= RARE_DEST_FLOWS_TO_SEE_TRAINING + && t_now - cli_lhost->getStartRareDestTraining() >= RARE_DEST_DURATION_TRAINING ) { - cli_host->setStartRareDestTraining(0); - cli_host->setRareDestLastEpoch(t_now); - ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training Off at %ld", t_now); + cli_lhost->setStartRareDestTraining(0); + cli_lhost->setRareDestLastEpoch(t_now); + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training Off at %ld ~ %s", t_now, host_id ); } - cli_host->dumpRareDestToRedis(); + cli_lhost->dumpRareDestToRedis(); return; } /* check epoch */ - if (t_now - cli_host->getRareDestLastEpoch() >= 2*RARE_DEST_EPOCH_DURATION) { + if (t_now - cli_lhost->getRareDestLastEpoch() >= 2*RARE_DEST_EPOCH_DURATION) { ndpi_bitmap_clear(rare_dest); ndpi_bitmap_clear(rare_dest_revise); - cli_host->dumpRareDestToRedis(); + cli_lhost->dumpRareDestToRedis(); return; } - if (t_now - cli_host->getRareDestLastEpoch() >= RARE_DEST_EPOCH_DURATION) { - //ndpi_bitmap_xor(rare_dest_revise, rare_dest); // updates rare_dest_revise + if (t_now - cli_lhost->getRareDestLastEpoch() >= RARE_DEST_EPOCH_DURATION) { + ndpi_bitmap_xor(rare_dest_revise, rare_dest); // updates rare_dest_revise ndpi_bitmap_and(rare_dest, rare_dest_revise); // makes rare_dest = rare_dest_revise - cli_host->setRareDestLastEpoch(t_now); + cli_lhost->setRareDestLastEpoch(t_now); } /* update */ @@ -123,7 +122,8 @@ void RareDestination::protocolDetected(Flow *f) { is_rare_destination = true; } - cli_host->dumpRareDestToRedis(); + cli_lhost->dumpRareDestToRedis(); + } if (is_rare_destination) { From ab8713d60cfe637d8029193d33cea11b742d5883 Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Fri, 19 May 2023 00:30:31 +0200 Subject: [PATCH 22/50] Commented Prints --- src/LocalHost.cpp | 7 ++----- src/flow_checks/RareDestination.cpp | 10 +++------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/LocalHost.cpp b/src/LocalHost.cpp index 487c4eb9f1e3..5bc083492e3a 100644 --- a/src/LocalHost.cpp +++ b/src/LocalHost.cpp @@ -598,7 +598,7 @@ void LocalHost::dumpRareDestToRedis() { snprintf(param_ser, sizeof(param_ser), "%lu", (unsigned long)rareDestTraining.seen); redis->hashSet(key, buf, param_ser); - //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data dumped to redis last_epoch: %ld RareDestTraining.start: %ld rareDestTraining.seen: %lu", last_epoch, rareDestTraining.start, (unsigned long)rareDestTraining.seen); + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data dumped for %s", get_hostkey(hostbuf, sizeof(hostbuf))); } bool LocalHost::loadRareDestFromRedis() { @@ -638,9 +638,6 @@ bool LocalHost::loadRareDestFromRedis() { return(false); } - /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "rare_dest size %lu", size); - ntop->getTrace()->traceEvent(TRACE_NORMAL, "rare_dest encoded: %s", value ); */ - rare_dest = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); free(value); @@ -666,7 +663,7 @@ bool LocalHost::loadRareDestFromRedis() { rare_dest_revise = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); free(value); - //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data loaded from redis , last_epoch: %ld RareDestTraining.start: %ld RareDestTraining.seen: %lu", last_epoch, rareDestTraining.start, rareDestTraining.seen); + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data loaded for %s", get_hostkey(hostbuf, sizeof(hostbuf))); return(true); } diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index f95331d07967..f9528a9d0d71 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -76,7 +76,7 @@ void RareDestination::protocolDetected(Flow *f) { if (!ndpi_bitmap_cardinality(rare_dest)) { cli_lhost->clearSeenRareDestTraining(); cli_lhost->setStartRareDestTraining(t_now); - ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training On at %ld ~ %s", t_now, host_id ); + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training On at %ld ~ %s", t_now, host_id ); } /* if host is training */ @@ -85,7 +85,7 @@ void RareDestination::protocolDetected(Flow *f) { if (!ndpi_bitmap_isset(rare_dest, hash)) { ndpi_bitmap_set(rare_dest, hash); cli_lhost->incrementSeenRareDestTraining(); - ntop->getTrace()->traceEvent(TRACE_NORMAL, "Hash %s added ~ %s", f->getFlowServerInfo(), host_id ); + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Hash %s added ~ %s", f->getFlowServerInfo(), host_id ); } /* check if training has to end */ if ( cli_lhost->getSeenRareDestTraining() >= RARE_DEST_FLOWS_TO_SEE_TRAINING @@ -93,9 +93,8 @@ void RareDestination::protocolDetected(Flow *f) { { cli_lhost->setStartRareDestTraining(0); cli_lhost->setRareDestLastEpoch(t_now); - ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training Off at %ld ~ %s", t_now, host_id ); + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training Off at %ld ~ %s", t_now, host_id ); } - cli_lhost->dumpRareDestToRedis(); return; } @@ -104,7 +103,6 @@ void RareDestination::protocolDetected(Flow *f) { if (t_now - cli_lhost->getRareDestLastEpoch() >= 2*RARE_DEST_EPOCH_DURATION) { ndpi_bitmap_clear(rare_dest); ndpi_bitmap_clear(rare_dest_revise); - cli_lhost->dumpRareDestToRedis(); return; } @@ -121,8 +119,6 @@ void RareDestination::protocolDetected(Flow *f) { ndpi_bitmap_set(rare_dest, hash); is_rare_destination = true; } - - cli_lhost->dumpRareDestToRedis(); } From 9c8411b78a3d87b4f13597afcff3711bfdc3400c Mon Sep 17 00:00:00 2001 From: LeoBubi Date: Fri, 19 May 2023 09:39:57 +0200 Subject: [PATCH 23/50] Code cleanup --- src/LocalHost.cpp | 3 --- src/flow_checks/RareDestination.cpp | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/LocalHost.cpp b/src/LocalHost.cpp index 5bc083492e3a..ec333e8aed80 100644 --- a/src/LocalHost.cpp +++ b/src/LocalHost.cpp @@ -150,9 +150,6 @@ void LocalHost::initialize() { if( !loadRareDestFromRedis() ){ rare_dest = ndpi_bitmap_alloc(); rare_dest_revise = ndpi_bitmap_alloc(); - last_epoch = time(NULL); - setStartRareDestTraining(time(NULL)); - clearSeenRareDestTraining(); } } diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index f9528a9d0d71..f0acb68a803c 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -35,13 +35,13 @@ u_int32_t RareDestination::getDestinationHash(Flow *f) { /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination MAC %u - %s", *hash, mac); */ } - if (dest->isIPv6() || dest->isIPv4()) { + else if (dest->isIPv6() || dest->isIPv4()) { char *ip = dest->get_ip()->print(buf,sizeof(buf)); hash = Utils::hashString(ip); /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination IPv6/IPv4 %u - %s", *hash, ip); */ } } - if (f->isLocalToRemote()) { + else if (f->isLocalToRemote()) { char name_buf[128]; char *domain = dest->get_name(name_buf, sizeof(name_buf), false); hash = Utils::hashString(domain); From e5795f4a53dfe066ee297bd11fb8723eba90e944 Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Fri, 19 May 2023 18:05:10 +0200 Subject: [PATCH 24/50] refactor: moved hashing function --- src/flow_checks/RareDestination.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index f0acb68a803c..51baac4a588a 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -59,15 +59,12 @@ void RareDestination::protocolDetected(Flow *f) { if(f->getFlowServerInfo() != NULL) { if(!f->get_cli_host()->isLocalHost()) return; - u_int32_t hash = getDestinationHash(f); - if(hash == 0) return; - LocalHost *cli_lhost = (LocalHost*)f->get_cli_host(); ndpi_bitmap *rare_dest = cli_lhost->getRareDestBMap(); ndpi_bitmap *rare_dest_revise = cli_lhost->getRareDestReviseBMap(); - char hostbuf[64], *host_id; - host_id = cli_lhost->get_hostkey(hostbuf, sizeof(hostbuf)); + /* char hostbuf[64], *host_id; + host_id = cli_lhost->get_hostkey(hostbuf, sizeof(hostbuf)); */ /* TODO: Check if bitmap pointers are valid */ time_t t_now = time(NULL); @@ -81,6 +78,8 @@ void RareDestination::protocolDetected(Flow *f) { /* if host is training */ if (cli_lhost->getStartRareDestTraining()) { + u_int32_t hash = getDestinationHash(f); + if(hash == 0) return; /* update */ if (!ndpi_bitmap_isset(rare_dest, hash)) { ndpi_bitmap_set(rare_dest, hash); @@ -88,8 +87,8 @@ void RareDestination::protocolDetected(Flow *f) { //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Hash %s added ~ %s", f->getFlowServerInfo(), host_id ); } /* check if training has to end */ - if ( cli_lhost->getSeenRareDestTraining() >= RARE_DEST_FLOWS_TO_SEE_TRAINING - && t_now - cli_lhost->getStartRareDestTraining() >= RARE_DEST_DURATION_TRAINING ) + if ( cli_lhost->getSeenRareDestTraining() >= 30 + && t_now - cli_lhost->getStartRareDestTraining() >= 180 /* RARE_DEST_DURATION_TRAINING */ ) { cli_lhost->setStartRareDestTraining(0); cli_lhost->setRareDestLastEpoch(t_now); @@ -100,18 +99,20 @@ void RareDestination::protocolDetected(Flow *f) { /* check epoch */ - if (t_now - cli_lhost->getRareDestLastEpoch() >= 2*RARE_DEST_EPOCH_DURATION) { + if (t_now - cli_lhost->getRareDestLastEpoch() >= 2*540 /* RARE_DEST_EPOCH_DURATION */) { ndpi_bitmap_clear(rare_dest); ndpi_bitmap_clear(rare_dest_revise); return; } - if (t_now - cli_lhost->getRareDestLastEpoch() >= RARE_DEST_EPOCH_DURATION) { + if (t_now - cli_lhost->getRareDestLastEpoch() >= 540 /* RARE_DEST_EPOCH_DURATION */) { ndpi_bitmap_xor(rare_dest_revise, rare_dest); // updates rare_dest_revise ndpi_bitmap_and(rare_dest, rare_dest_revise); // makes rare_dest = rare_dest_revise cli_lhost->setRareDestLastEpoch(t_now); } + u_int32_t hash = getDestinationHash(f); + if(hash == 0) return; /* update */ if (ndpi_bitmap_isset(rare_dest, hash)) ndpi_bitmap_unset(rare_dest_revise, hash); From 300df1d14f674aef6903be7b58a47c6e079d746b Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Thu, 13 Jul 2023 16:17:52 +0200 Subject: [PATCH 25/50] test: simplified hash function --- src/LocalHost.cpp | 5 ++--- src/flow_checks/RareDestination.cpp | 20 +++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/LocalHost.cpp b/src/LocalHost.cpp index ec333e8aed80..015c1b562351 100644 --- a/src/LocalHost.cpp +++ b/src/LocalHost.cpp @@ -595,7 +595,7 @@ void LocalHost::dumpRareDestToRedis() { snprintf(param_ser, sizeof(param_ser), "%lu", (unsigned long)rareDestTraining.seen); redis->hashSet(key, buf, param_ser); - ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data dumped for %s", get_hostkey(hostbuf, sizeof(hostbuf))); + /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data dumped for %s", get_hostkey(hostbuf, sizeof(hostbuf))); */ } bool LocalHost::loadRareDestFromRedis() { @@ -660,7 +660,6 @@ bool LocalHost::loadRareDestFromRedis() { rare_dest_revise = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); free(value); - ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data loaded for %s", get_hostkey(hostbuf, sizeof(hostbuf))); - + /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data loaded for %s", get_hostkey(hostbuf, sizeof(hostbuf))); */ return(true); } diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index 51baac4a588a..7c30097c7d01 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -28,16 +28,18 @@ u_int32_t RareDestination::getDestinationHash(Flow *f) { u_int32_t hash = 0; Host *dest = f->get_srv_host(); if (f->isLocalToLocal()) { - char buf[64]; + /* char buf[64]; */ if (!dest->isMulticastHost() && dest->isDHCPHost()) { - char *mac = dest->getMac()->print(buf,sizeof(buf)); - hash = Utils::hashString(mac); + u_int32_t mac = dest->getMac()->key(); + hash = mac; + /* char *mac = dest->getMac()->print(buf,sizeof(buf)); + hash = Utils::hashString(mac); */ /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination MAC %u - %s", *hash, mac); */ } - else if (dest->isIPv6() || dest->isIPv4()) { - char *ip = dest->get_ip()->print(buf,sizeof(buf)); - hash = Utils::hashString(ip); + hash = dest->key(); + /* char *ip = dest->get_ip()->print(buf,sizeof(buf)); + hash = Utils::hashString(ip); */ /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination IPv6/IPv4 %u - %s", *hash, ip); */ } } @@ -88,7 +90,7 @@ void RareDestination::protocolDetected(Flow *f) { } /* check if training has to end */ if ( cli_lhost->getSeenRareDestTraining() >= 30 - && t_now - cli_lhost->getStartRareDestTraining() >= 180 /* RARE_DEST_DURATION_TRAINING */ ) + && t_now - cli_lhost->getStartRareDestTraining() >= 300 /* RARE_DEST_DURATION_TRAINING */ ) { cli_lhost->setStartRareDestTraining(0); cli_lhost->setRareDestLastEpoch(t_now); @@ -99,13 +101,13 @@ void RareDestination::protocolDetected(Flow *f) { /* check epoch */ - if (t_now - cli_lhost->getRareDestLastEpoch() >= 2*540 /* RARE_DEST_EPOCH_DURATION */) { + if (t_now - cli_lhost->getRareDestLastEpoch() >= 2*600 /* RARE_DEST_EPOCH_DURATION */) { ndpi_bitmap_clear(rare_dest); ndpi_bitmap_clear(rare_dest_revise); return; } - if (t_now - cli_lhost->getRareDestLastEpoch() >= 540 /* RARE_DEST_EPOCH_DURATION */) { + if (t_now - cli_lhost->getRareDestLastEpoch() >= 600 /* RARE_DEST_EPOCH_DURATION */) { ndpi_bitmap_xor(rare_dest_revise, rare_dest); // updates rare_dest_revise ndpi_bitmap_and(rare_dest, rare_dest_revise); // makes rare_dest = rare_dest_revise cli_lhost->setRareDestLastEpoch(t_now); From 967e1824bf253b7cb9e4b187051bf9b95155096b Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Tue, 18 Jul 2023 17:28:13 +0200 Subject: [PATCH 26/50] Cleanup RareDestination --- src/flow_checks/RareDestination.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index 7c30097c7d01..e8f7b2ebdfb7 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -27,28 +27,34 @@ u_int32_t RareDestination::getDestinationHash(Flow *f) { u_int32_t hash = 0; Host *dest = f->get_srv_host(); + if (f->isLocalToLocal()) { /* char buf[64]; */ if (!dest->isMulticastHost() && dest->isDHCPHost()) { u_int32_t mac = dest->getMac()->key(); hash = mac; + /* char *mac = dest->getMac()->print(buf,sizeof(buf)); hash = Utils::hashString(mac); */ + /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination MAC %u - %s", *hash, mac); */ } else if (dest->isIPv6() || dest->isIPv4()) { hash = dest->key(); + /* char *ip = dest->get_ip()->print(buf,sizeof(buf)); hash = Utils::hashString(ip); */ + /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination IPv6/IPv4 %u - %s", *hash, ip); */ } - } - else if (f->isLocalToRemote()) { + } else if (f->isLocalToRemote()) { char name_buf[128]; char *domain = dest->get_name(name_buf, sizeof(name_buf), false); hash = Utils::hashString(domain); + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare remote destination %u - %s - %s", hash, domain, dest->getServerName(name_buf, sizeof(name_buf))); } + return hash; } @@ -64,9 +70,7 @@ void RareDestination::protocolDetected(Flow *f) { LocalHost *cli_lhost = (LocalHost*)f->get_cli_host(); ndpi_bitmap *rare_dest = cli_lhost->getRareDestBMap(); ndpi_bitmap *rare_dest_revise = cli_lhost->getRareDestReviseBMap(); - - /* char hostbuf[64], *host_id; - host_id = cli_lhost->get_hostkey(hostbuf, sizeof(hostbuf)); */ + /* TODO: Check if bitmap pointers are valid */ time_t t_now = time(NULL); @@ -82,6 +86,7 @@ void RareDestination::protocolDetected(Flow *f) { if (cli_lhost->getStartRareDestTraining()) { u_int32_t hash = getDestinationHash(f); if(hash == 0) return; + /* update */ if (!ndpi_bitmap_isset(rare_dest, hash)) { ndpi_bitmap_set(rare_dest, hash); @@ -96,6 +101,7 @@ void RareDestination::protocolDetected(Flow *f) { cli_lhost->setRareDestLastEpoch(t_now); //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training Off at %ld ~ %s", t_now, host_id ); } + return; } From 1dd4ec7d3c64b1ae60a91bc7ad756c95b4eb1367 Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Tue, 25 Jul 2023 02:37:45 +0200 Subject: [PATCH 27/50] periodic training implemented + minor performance enhancing --- include/LocalHost.h | 24 +++++----- include/ntop_defines.h | 7 ++- src/LocalHost.cpp | 54 ++++++++++----------- src/flow_checks/RareDestination.cpp | 73 ++++++++++++++--------------- 4 files changed, 77 insertions(+), 81 deletions(-) diff --git a/include/LocalHost.h b/include/LocalHost.h index 839236c04086..60440be9e8a6 100644 --- a/include/LocalHost.h +++ b/include/LocalHost.h @@ -38,12 +38,12 @@ class LocalHost : public Host { /* RareDestination data implementation*/ ndpi_bitmap *rare_dest; - ndpi_bitmap *rare_dest_revise; - time_t last_epoch; + ndpi_bitmap *rare_dest_last; struct { time_t start; - u_int32_t seen; + time_t last_training; + bool training; } rareDestTraining; /* LocalHost data: update LocalHost::deleteHostData when adding new fields */ @@ -194,18 +194,20 @@ class LocalHost : public Host { /* RareDest Extension methods */ - inline ndpi_bitmap* getRareDestBMap() const { return(rare_dest); } - inline ndpi_bitmap* getRareDestReviseBMap() const { return(rare_dest_revise); } - - inline time_t getRareDestLastEpoch() const { return(last_epoch); } - inline void setRareDestLastEpoch(time_t t) { last_epoch=t; } + inline ndpi_bitmap* getRareDestBitmap() const { return(rare_dest); } inline time_t getStartRareDestTraining() const { return(rareDestTraining.start); } inline void setStartRareDestTraining(time_t t) { rareDestTraining.start = t; } - inline u_int32_t getSeenRareDestTraining() const { return(rareDestTraining.seen); } - inline void clearSeenRareDestTraining() { rareDestTraining.seen = 0; } - inline void incrementSeenRareDestTraining() { rareDestTraining.seen++; } + inline time_t getLastRareDestTraining() const { return(rareDestTraining.last_training); } + inline void setLastRareDestTraining(time_t t) { rareDestTraining.last_training = t; } + + inline bool isTrainingRareDest() const { return(rareDestTraining.training); } + inline void startRareDestTraining() { rareDestTraining.training = true; } + inline void stopRareDestTraining() { rareDestTraining.training = false; } + + inline void addRareDestBitmaps() { ndpi_bitmap_or(rare_dest, rare_dest_last); } + inline void mergeRareDestBitmaps() { ndpi_bitmap_cardinality(rare_dest_last) ? ndpi_bitmap_and(rare_dest_last, rare_dest) : ndpi_bitmap_or(rare_dest_last, rare_dest); ndpi_bitmap_clear(rare_dest); } void dumpRareDestToRedis(); bool loadRareDestFromRedis(); diff --git a/include/ntop_defines.h b/include/ntop_defines.h index 729f445e8fe0..3afd945a3c0b 100644 --- a/include/ntop_defines.h +++ b/include/ntop_defines.h @@ -1487,10 +1487,9 @@ extern struct ntopngLuaContext *getUserdata(struct lua_State *vm); /******************************************************************************/ -#define HOST_RARE_DEST_SERIALIZED_KEY "ntopng.localhost_rare_dest_fields.%s" -#define RARE_DEST_DURATION_TRAINING 60 /* seconds */ -#define RARE_DEST_FLOWS_TO_SEE_TRAINING 10 -#define RARE_DEST_EPOCH_DURATION 3600 /* seconds */ +#define HOST_RARE_DEST_SERIALIZED_KEY "ntopng.localhost_rare_dest_fields.%u" +#define RARE_DEST_DURATION_TRAINING 3600 /* seconds ( 1 hour ) */ +#define RARE_DEST_LAST_TRAINING_GAP 7200 /* seconds ( 2 hours ) */ /******************************************************************************/ diff --git a/src/LocalHost.cpp b/src/LocalHost.cpp index 015c1b562351..c56db4c6656d 100644 --- a/src/LocalHost.cpp +++ b/src/LocalHost.cpp @@ -50,7 +50,7 @@ LocalHost::~LocalHost() { dumpRareDestToRedis(); if(rare_dest) ndpi_bitmap_free(rare_dest); - if(rare_dest_revise) ndpi_bitmap_free(rare_dest_revise); + if(rare_dest_last) ndpi_bitmap_free(rare_dest_last); freeLocalHostData(); } @@ -148,8 +148,9 @@ void LocalHost::initialize() { #endif if( !loadRareDestFromRedis() ){ + rareDestTraining.start = 0; rare_dest = ndpi_bitmap_alloc(); - rare_dest_revise = ndpi_bitmap_alloc(); + rare_dest_last = ndpi_bitmap_alloc(); } } @@ -548,10 +549,10 @@ void LocalHost::dumpRareDestToRedis() { Redis *redis = ntop->getRedis(); size_t size; - if((!redis) || (!rare_dest) || (!rare_dest_revise)) return; - - char hostbuf[64]; - snprintf(key, sizeof(key), HOST_RARE_DEST_SERIALIZED_KEY, get_hostkey(hostbuf, sizeof(hostbuf))); + if((!redis) || (!rare_dest) || (!rare_dest_last)) return; + + getSerializationKey(key, sizeof(key)); + //snprintf(key, sizeof(key), HOST_RARE_DEST_SERIALIZED_KEY, iface->get_id()); snprintf(buf, sizeof(buf), "rare_dest"); size = ndpi_bitmap_serialize(rare_dest, &value); @@ -567,8 +568,8 @@ void LocalHost::dumpRareDestToRedis() { snprintf(param_ser, sizeof(param_ser), "%lu", size); redis->hashSet(key, buf, param_ser); - snprintf(buf, sizeof(buf), "rare_dest_revise"); - size = ndpi_bitmap_serialize(rare_dest_revise, &value); + snprintf(buf, sizeof(buf), "rare_dest_last"); + size = ndpi_bitmap_serialize(rare_dest_last, &value); if (value) { char *encoded_bmap = Utils::base64_encode((unsigned char *)value, size); @@ -578,21 +579,20 @@ void LocalHost::dumpRareDestToRedis() { free(value); } - snprintf(buf, sizeof(buf), "rare_dest_revise_len"); + snprintf(buf, sizeof(buf), "rare_dest_last_len"); snprintf(param_ser, sizeof(param_ser), "%lu", size); redis->hashSet(key, buf, param_ser); - - snprintf(buf, sizeof(buf), "last_epoch"); - snprintf(param_ser, sizeof(param_ser), "%ld", last_epoch); - redis->hashSet(key, buf, param_ser); - snprintf(buf, sizeof(buf), "t_start_training"); snprintf(param_ser, sizeof(param_ser), "%ld", rareDestTraining.start); redis->hashSet(key, buf, param_ser); - snprintf(buf, sizeof(buf), "seen_flows_training"); - snprintf(param_ser, sizeof(param_ser), "%lu", (unsigned long)rareDestTraining.seen); + snprintf(buf, sizeof(buf), "last_training"); + snprintf(param_ser, sizeof(param_ser), "%ld", rareDestTraining.last_training); + redis->hashSet(key, buf, param_ser); + + snprintf(buf, sizeof(buf), "training"); + snprintf(param_ser, sizeof(param_ser), "%d", rareDestTraining.training); redis->hashSet(key, buf, param_ser); /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data dumped for %s", get_hostkey(hostbuf, sizeof(hostbuf))); */ @@ -605,20 +605,20 @@ bool LocalHost::loadRareDestFromRedis() { if((!redis)) return(false); - char hostbuf[64]; - snprintf(key, sizeof(key), HOST_RARE_DEST_SERIALIZED_KEY, get_hostkey(hostbuf, sizeof(hostbuf))); - - snprintf(buf, sizeof(buf), "last_epoch"); - if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); - last_epoch = atol(param_str); + getSerializationKey(key, sizeof(key)); + //snprintf(key, sizeof(key), HOST_RARE_DEST_SERIALIZED_KEY, iface->get_id()); snprintf(buf, sizeof(buf), "t_start_training"); if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); rareDestTraining.start = atol(param_str); - snprintf(buf, sizeof(buf), "seen_flows_training"); + snprintf(buf, sizeof(buf), "last_training"); + if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); + rareDestTraining.last_training = atol(param_str); + + snprintf(buf, sizeof(buf), "training"); if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); - rareDestTraining.seen = (u_int32_t)strtoul(param_str, NULL, 10); + rareDestTraining.training = (bool)atoi(param_str); snprintf(buf, sizeof(buf), "rare_dest_len"); if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); @@ -638,7 +638,7 @@ bool LocalHost::loadRareDestFromRedis() { rare_dest = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); free(value); - snprintf(buf, sizeof(buf), "rare_dest_revise_len"); + snprintf(buf, sizeof(buf), "rare_dest_last_len"); if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) { ndpi_bitmap_free(rare_dest); return(false); @@ -651,13 +651,13 @@ bool LocalHost::loadRareDestFromRedis() { return(false); } - snprintf(buf, sizeof(buf), "rare_dest_revise"); + snprintf(buf, sizeof(buf), "rare_dest_last"); if(redis->hashGet(key, buf, value, size) != 0) { free(value); ndpi_bitmap_free(rare_dest); return(false); } - rare_dest_revise = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); + rare_dest_last = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); free(value); /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data loaded for %s", get_hostkey(hostbuf, sizeof(hostbuf))); */ diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index e8f7b2ebdfb7..e3d4b7bfff4e 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -29,23 +29,23 @@ u_int32_t RareDestination::getDestinationHash(Flow *f) { Host *dest = f->get_srv_host(); if (f->isLocalToLocal()) { - /* char buf[64]; */ + char buf[64]; if (!dest->isMulticastHost() && dest->isDHCPHost()) { - u_int32_t mac = dest->getMac()->key(); - hash = mac; + /* u_int32_t mac = dest->getMac()->key(); + hash = mac; */ - /* char *mac = dest->getMac()->print(buf,sizeof(buf)); - hash = Utils::hashString(mac); */ + char *mac = dest->getMac()->print(buf,sizeof(buf)); + hash = Utils::hashString(mac); - /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination MAC %u - %s", *hash, mac); */ + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination MAC %u - %s", hash, mac); } else if (dest->isIPv6() || dest->isIPv4()) { - hash = dest->key(); + //hash = dest->key(); - /* char *ip = dest->get_ip()->print(buf,sizeof(buf)); - hash = Utils::hashString(ip); */ + char *ip = dest->get_ip()->print(buf,sizeof(buf)); + hash = Utils::hashString(ip); - /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination IPv6/IPv4 %u - %s", *hash, ip); */ + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination IPv6/IPv4 %u - %s", hash, ip); } } else if (f->isLocalToRemote()) { char name_buf[128]; @@ -68,64 +68,59 @@ void RareDestination::protocolDetected(Flow *f) { if(!f->get_cli_host()->isLocalHost()) return; LocalHost *cli_lhost = (LocalHost*)f->get_cli_host(); - ndpi_bitmap *rare_dest = cli_lhost->getRareDestBMap(); - ndpi_bitmap *rare_dest_revise = cli_lhost->getRareDestReviseBMap(); + ndpi_bitmap *rare_dest = cli_lhost->getRareDestBitmap(); - /* TODO: Check if bitmap pointers are valid */ + /* char hostbuf[64], *host_id; + host_id = cli_lhost->get_hostkey(hostbuf, sizeof(hostbuf)); */ time_t t_now = time(NULL); /* check if training has to start */ - if (!ndpi_bitmap_cardinality(rare_dest)) { - cli_lhost->clearSeenRareDestTraining(); + if (!cli_lhost->getStartRareDestTraining()) { + cli_lhost->startRareDestTraining(); cli_lhost->setStartRareDestTraining(t_now); //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training On at %ld ~ %s", t_now, host_id ); } /* if host is training */ - if (cli_lhost->getStartRareDestTraining()) { + if (cli_lhost->isTrainingRareDest()) { u_int32_t hash = getDestinationHash(f); - if(hash == 0) return; + /* if(hash == 0) return; */ /* update */ - if (!ndpi_bitmap_isset(rare_dest, hash)) { + if ( hash && !ndpi_bitmap_isset(rare_dest, hash)) { ndpi_bitmap_set(rare_dest, hash); - cli_lhost->incrementSeenRareDestTraining(); //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Hash %s added ~ %s", f->getFlowServerInfo(), host_id ); } + /* check if training has to end */ - if ( cli_lhost->getSeenRareDestTraining() >= 30 - && t_now - cli_lhost->getStartRareDestTraining() >= 300 /* RARE_DEST_DURATION_TRAINING */ ) + if (t_now - cli_lhost->getStartRareDestTraining() >= 300 /* 1h RARE_DEST_DURATION_TRAINING */ ) { - cli_lhost->setStartRareDestTraining(0); - cli_lhost->setRareDestLastEpoch(t_now); + cli_lhost->stopRareDestTraining(); + cli_lhost->setLastRareDestTraining(t_now); + + cli_lhost->addRareDestBitmaps(); //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training Off at %ld ~ %s", t_now, host_id ); } return; } - /* check epoch */ - - if (t_now - cli_lhost->getRareDestLastEpoch() >= 2*600 /* RARE_DEST_EPOCH_DURATION */) { - ndpi_bitmap_clear(rare_dest); - ndpi_bitmap_clear(rare_dest_revise); + if ( t_now - cli_lhost->getLastRareDestTraining() > 600 /* 10min RARE_DEST_LAST_TRAINING_GAP */ ) + { + cli_lhost->mergeRareDestBitmaps(); + cli_lhost->setStartRareDestTraining(0); + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Merged Bitmaps %s", host_id ); return; } - - if (t_now - cli_lhost->getRareDestLastEpoch() >= 600 /* RARE_DEST_EPOCH_DURATION */) { - ndpi_bitmap_xor(rare_dest_revise, rare_dest); // updates rare_dest_revise - ndpi_bitmap_and(rare_dest, rare_dest_revise); // makes rare_dest = rare_dest_revise - cli_lhost->setRareDestLastEpoch(t_now); - } + u_int32_t hash = getDestinationHash(f); - if(hash == 0) return; + /* if(hash == 0) return; */ + /* update */ - if (ndpi_bitmap_isset(rare_dest, hash)) - ndpi_bitmap_unset(rare_dest_revise, hash); - else { - ndpi_bitmap_set(rare_dest, hash); + if (hash && !ndpi_bitmap_isset(rare_dest, hash)){ + ndpi_bitmap_set(rare_dest, hash); // Fire an alarm only once is_rare_destination = true; } From 841c2cb98d38f5141efdfbc9cd85fc6da6f96e52 Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Wed, 26 Jul 2023 14:48:48 +0200 Subject: [PATCH 28/50] simplified hash and training functions --- src/LocalHost.cpp | 16 +++++++---- src/flow_checks/RareDestination.cpp | 44 +++++++++-------------------- 2 files changed, 24 insertions(+), 36 deletions(-) diff --git a/src/LocalHost.cpp b/src/LocalHost.cpp index c56db4c6656d..afcee3d5f83f 100644 --- a/src/LocalHost.cpp +++ b/src/LocalHost.cpp @@ -149,6 +149,8 @@ void LocalHost::initialize() { if( !loadRareDestFromRedis() ){ rareDestTraining.start = 0; + rareDestTraining.last_training = 0; + rareDestTraining.training = false; rare_dest = ndpi_bitmap_alloc(); rare_dest_last = ndpi_bitmap_alloc(); } @@ -551,8 +553,8 @@ void LocalHost::dumpRareDestToRedis() { if((!redis) || (!rare_dest) || (!rare_dest_last)) return; - getSerializationKey(key, sizeof(key)); - //snprintf(key, sizeof(key), HOST_RARE_DEST_SERIALIZED_KEY, iface->get_id()); + char hostbuf[64]; + snprintf(key, sizeof(key), HOST_RARE_DEST_SERIALIZED_KEY, get_hostkey(hostbuf, sizeof(hostbuf))); snprintf(buf, sizeof(buf), "rare_dest"); size = ndpi_bitmap_serialize(rare_dest, &value); @@ -592,7 +594,7 @@ void LocalHost::dumpRareDestToRedis() { redis->hashSet(key, buf, param_ser); snprintf(buf, sizeof(buf), "training"); - snprintf(param_ser, sizeof(param_ser), "%d", rareDestTraining.training); + snprintf(param_ser, sizeof(param_ser), "%d", rareDestTraining.training ? 1 : 0); redis->hashSet(key, buf, param_ser); /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data dumped for %s", get_hostkey(hostbuf, sizeof(hostbuf))); */ @@ -605,8 +607,8 @@ bool LocalHost::loadRareDestFromRedis() { if((!redis)) return(false); - getSerializationKey(key, sizeof(key)); - //snprintf(key, sizeof(key), HOST_RARE_DEST_SERIALIZED_KEY, iface->get_id()); + char hostbuf[64]; + snprintf(key, sizeof(key), HOST_RARE_DEST_SERIALIZED_KEY, get_hostkey(hostbuf, sizeof(hostbuf))); snprintf(buf, sizeof(buf), "t_start_training"); if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); @@ -618,7 +620,9 @@ bool LocalHost::loadRareDestFromRedis() { snprintf(buf, sizeof(buf), "training"); if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); - rareDestTraining.training = (bool)atoi(param_str); + rareDestTraining.training = atoi(param_str) ? true : false; + + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training loaded %d", rareDestTraining.training); snprintf(buf, sizeof(buf), "rare_dest_len"); if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index e3d4b7bfff4e..df69f0352f02 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -28,33 +28,23 @@ u_int32_t RareDestination::getDestinationHash(Flow *f) { u_int32_t hash = 0; Host *dest = f->get_srv_host(); - if (f->isLocalToLocal()) { + if (f->isLocalToLocal() || f->isLocalToRemote()) { char buf[64]; if (!dest->isMulticastHost() && dest->isDHCPHost()) { - /* u_int32_t mac = dest->getMac()->key(); - hash = mac; */ char *mac = dest->getMac()->print(buf,sizeof(buf)); hash = Utils::hashString(mac); - //ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination MAC %u - %s", hash, mac); + /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination MAC %u - %s", *hash, mac); */ } - else if (dest->isIPv6() || dest->isIPv4()) { - //hash = dest->key(); + else /* if (dest->isIPv6() || dest->isIPv4()) */ { char *ip = dest->get_ip()->print(buf,sizeof(buf)); hash = Utils::hashString(ip); - //ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination IPv6/IPv4 %u - %s", hash, ip); + /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination IPv6/IPv4 %u - %s", *hash, ip); */ } - } else if (f->isLocalToRemote()) { - char name_buf[128]; - char *domain = dest->get_name(name_buf, sizeof(name_buf), false); - hash = Utils::hashString(domain); - - //ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare remote destination %u - %s - %s", hash, domain, dest->getServerName(name_buf, sizeof(name_buf))); } - return hash; } @@ -82,16 +72,13 @@ void RareDestination::protocolDetected(Flow *f) { //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training On at %ld ~ %s", t_now, host_id ); } - /* if host is training */ - if (cli_lhost->isTrainingRareDest()) { - u_int32_t hash = getDestinationHash(f); - /* if(hash == 0) return; */ + u_int32_t hash = getDestinationHash(f); + if(hash == 0) return; - /* update */ - if ( hash && !ndpi_bitmap_isset(rare_dest, hash)) { - ndpi_bitmap_set(rare_dest, hash); - //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Hash %s added ~ %s", f->getFlowServerInfo(), host_id ); - } + /* training */ + if (cli_lhost->isTrainingRareDest()) { + ndpi_bitmap_set(rare_dest, hash); + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Hash %s added ~ %s", f->getFlowServerInfo(), host_id ); /* check if training has to end */ if (t_now - cli_lhost->getStartRareDestTraining() >= 300 /* 1h RARE_DEST_DURATION_TRAINING */ ) @@ -99,7 +86,7 @@ void RareDestination::protocolDetected(Flow *f) { cli_lhost->stopRareDestTraining(); cli_lhost->setLastRareDestTraining(t_now); - cli_lhost->addRareDestBitmaps(); + //cli_lhost->addRareDestBitmaps(); //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training Off at %ld ~ %s", t_now, host_id ); } @@ -108,18 +95,15 @@ void RareDestination::protocolDetected(Flow *f) { if ( t_now - cli_lhost->getLastRareDestTraining() > 600 /* 10min RARE_DEST_LAST_TRAINING_GAP */ ) { - cli_lhost->mergeRareDestBitmaps(); + //cli_lhost->mergeRareDestBitmaps(); cli_lhost->setStartRareDestTraining(0); + ndpi_bitmap_clear(rare_dest); //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Merged Bitmaps %s", host_id ); return; } - - - u_int32_t hash = getDestinationHash(f); - /* if(hash == 0) return; */ /* update */ - if (hash && !ndpi_bitmap_isset(rare_dest, hash)){ + if (!ndpi_bitmap_isset(rare_dest, hash)){ ndpi_bitmap_set(rare_dest, hash); // Fire an alarm only once is_rare_destination = true; } From 6979e15fed722bc74413918e304efe143ec1752a Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Wed, 26 Jul 2023 18:09:47 +0200 Subject: [PATCH 29/50] improved performance tweaks --- include/ntop_defines.h | 2 +- src/LocalHost.cpp | 2 -- src/flow_checks/RareDestination.cpp | 21 +++++++++++++-------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/include/ntop_defines.h b/include/ntop_defines.h index 3afd945a3c0b..4b606aacb564 100644 --- a/include/ntop_defines.h +++ b/include/ntop_defines.h @@ -1487,7 +1487,7 @@ extern struct ntopngLuaContext *getUserdata(struct lua_State *vm); /******************************************************************************/ -#define HOST_RARE_DEST_SERIALIZED_KEY "ntopng.localhost_rare_dest_fields.%u" +#define HOST_RARE_DEST_SERIALIZED_KEY "ntopng.localhost_rare_dest_fields.%s" #define RARE_DEST_DURATION_TRAINING 3600 /* seconds ( 1 hour ) */ #define RARE_DEST_LAST_TRAINING_GAP 7200 /* seconds ( 2 hours ) */ diff --git a/src/LocalHost.cpp b/src/LocalHost.cpp index afcee3d5f83f..a40a847c383e 100644 --- a/src/LocalHost.cpp +++ b/src/LocalHost.cpp @@ -622,8 +622,6 @@ bool LocalHost::loadRareDestFromRedis() { if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); rareDestTraining.training = atoi(param_str) ? true : false; - ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training loaded %d", rareDestTraining.training); - snprintf(buf, sizeof(buf), "rare_dest_len"); if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); size = (size_t)strtoul(param_str, NULL, 10); diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index df69f0352f02..b76927cf2960 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -28,7 +28,7 @@ u_int32_t RareDestination::getDestinationHash(Flow *f) { u_int32_t hash = 0; Host *dest = f->get_srv_host(); - if (f->isLocalToLocal() || f->isLocalToRemote()) { + if (f->isLocalToLocal()) { char buf[64]; if (!dest->isMulticastHost() && dest->isDHCPHost()) { @@ -37,14 +37,17 @@ u_int32_t RareDestination::getDestinationHash(Flow *f) { /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination MAC %u - %s", *hash, mac); */ } - else /* if (dest->isIPv6() || dest->isIPv4()) */ { + else if (dest->isIPv6() || dest->isIPv4()) { char *ip = dest->get_ip()->print(buf,sizeof(buf)); hash = Utils::hashString(ip); /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination IPv6/IPv4 %u - %s", *hash, ip); */ } + } else if (f->isLocalToRemote()){ + hash = Utils::hashString(f->getFlowServerInfo()); } + return hash; } @@ -54,8 +57,7 @@ void RareDestination::protocolDetected(Flow *f) { bool is_rare_destination = false; - if(f->getFlowServerInfo() != NULL) { - if(!f->get_cli_host()->isLocalHost()) return; + if(f->getFlowServerInfo() != NULL && f->get_cli_host()->isLocalHost()) { LocalHost *cli_lhost = (LocalHost*)f->get_cli_host(); ndpi_bitmap *rare_dest = cli_lhost->getRareDestBitmap(); @@ -81,23 +83,26 @@ void RareDestination::protocolDetected(Flow *f) { //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Hash %s added ~ %s", f->getFlowServerInfo(), host_id ); /* check if training has to end */ - if (t_now - cli_lhost->getStartRareDestTraining() >= 300 /* 1h RARE_DEST_DURATION_TRAINING */ ) + if (t_now - cli_lhost->getStartRareDestTraining() >= 300 /* RARE_DEST_DURATION_TRAINING */ ) { + cli_lhost->addRareDestBitmaps(); + cli_lhost->stopRareDestTraining(); cli_lhost->setLastRareDestTraining(t_now); - //cli_lhost->addRareDestBitmaps(); //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training Off at %ld ~ %s", t_now, host_id ); } return; } - if ( t_now - cli_lhost->getLastRareDestTraining() > 600 /* 10min RARE_DEST_LAST_TRAINING_GAP */ ) + if ( t_now - cli_lhost->getLastRareDestTraining() > 600 /* RARE_DEST_LAST_TRAINING_GAP */ ) { - //cli_lhost->mergeRareDestBitmaps(); + cli_lhost->mergeRareDestBitmaps(); + cli_lhost->setStartRareDestTraining(0); ndpi_bitmap_clear(rare_dest); + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Merged Bitmaps %s", host_id ); return; } From 637460d28498e258f04274d1cc0f5e4fffe581cd Mon Sep 17 00:00:00 2001 From: Leonardo Brugnano Date: Fri, 28 Jul 2023 18:31:43 +0200 Subject: [PATCH 30/50] RareDestination minor fix --- src/flow_checks/RareDestination.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index b76927cf2960..2cf6d03ed756 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -86,7 +86,6 @@ void RareDestination::protocolDetected(Flow *f) { if (t_now - cli_lhost->getStartRareDestTraining() >= 300 /* RARE_DEST_DURATION_TRAINING */ ) { cli_lhost->addRareDestBitmaps(); - cli_lhost->stopRareDestTraining(); cli_lhost->setLastRareDestTraining(t_now); @@ -99,9 +98,8 @@ void RareDestination::protocolDetected(Flow *f) { if ( t_now - cli_lhost->getLastRareDestTraining() > 600 /* RARE_DEST_LAST_TRAINING_GAP */ ) { cli_lhost->mergeRareDestBitmaps(); - cli_lhost->setStartRareDestTraining(0); - ndpi_bitmap_clear(rare_dest); + // ndpi_bitmap_clear(rare_dest); useless, see mergeRareDestBitmaps //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Merged Bitmaps %s", host_id ); return; From 8b33fc909c7a76f5aa40a541ba4cdd63501292ca Mon Sep 17 00:00:00 2001 From: Leonardo Brugnano Date: Fri, 28 Jul 2023 20:15:05 +0200 Subject: [PATCH 31/50] Fixed incorrect behavior and code cleanup --- include/LocalHost.h | 7 +++-- src/flow_checks/RareDestination.cpp | 47 ++++++++++++++++++----------- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/include/LocalHost.h b/include/LocalHost.h index 60440be9e8a6..9ed30a0f8cd2 100644 --- a/include/LocalHost.h +++ b/include/LocalHost.h @@ -194,7 +194,8 @@ class LocalHost : public Host { /* RareDest Extension methods */ - inline ndpi_bitmap* getRareDestBitmap() const { return(rare_dest); } + inline ndpi_bitmap* getRareDestBitmap() const { return(rare_dest); } + inline ndpi_bitmap* getRareDestLastBitmap() const { return(rare_dest_last); } inline time_t getStartRareDestTraining() const { return(rareDestTraining.start); } inline void setStartRareDestTraining(time_t t) { rareDestTraining.start = t; } @@ -206,8 +207,8 @@ class LocalHost : public Host { inline void startRareDestTraining() { rareDestTraining.training = true; } inline void stopRareDestTraining() { rareDestTraining.training = false; } - inline void addRareDestBitmaps() { ndpi_bitmap_or(rare_dest, rare_dest_last); } - inline void mergeRareDestBitmaps() { ndpi_bitmap_cardinality(rare_dest_last) ? ndpi_bitmap_and(rare_dest_last, rare_dest) : ndpi_bitmap_or(rare_dest_last, rare_dest); ndpi_bitmap_clear(rare_dest); } + inline void updateRareDestLastBitmap() { ndpi_bitmap_clear(rare_dest_last); ndpi_bitmap_or(rare_dest_last, rare_dest); } + inline void clearRareDestBitmaps() { ndpi_bitmap_clear(rare_dest); ndpi_bitmap_clear(rare_dest_last); } void dumpRareDestToRedis(); bool loadRareDestFromRedis(); diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index 2cf6d03ed756..86bf4a0b0766 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -25,26 +25,28 @@ /* ***************************************************** */ u_int32_t RareDestination::getDestinationHash(Flow *f) { + u_int32_t hash = 0; Host *dest = f->get_srv_host(); if (f->isLocalToLocal()) { + char buf[64]; + if (!dest->isMulticastHost() && dest->isDHCPHost()) { - char *mac = dest->getMac()->print(buf,sizeof(buf)); hash = Utils::hashString(mac); - /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination MAC %u - %s", *hash, mac); */ } - else if (dest->isIPv6() || dest->isIPv4()) { + else if (dest->isIPv6() || dest->isIPv4()) { char *ip = dest->get_ip()->print(buf,sizeof(buf)); hash = Utils::hashString(ip); - /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination IPv6/IPv4 %u - %s", *hash, ip); */ } - } else if (f->isLocalToRemote()){ + } + + else if (f->isLocalToRemote()){ hash = Utils::hashString(f->getFlowServerInfo()); } @@ -59,8 +61,9 @@ void RareDestination::protocolDetected(Flow *f) { if(f->getFlowServerInfo() != NULL && f->get_cli_host()->isLocalHost()) { - LocalHost *cli_lhost = (LocalHost*)f->get_cli_host(); + LocalHost *cli_lhost = (LocalHost*)f->get_cli_host(); ndpi_bitmap *rare_dest = cli_lhost->getRareDestBitmap(); + ndpi_bitmap *rare_dest_last = cli_lhost->getRareDestLastBitmap(); /* char hostbuf[64], *host_id; host_id = cli_lhost->get_hostkey(hostbuf, sizeof(hostbuf)); */ @@ -77,40 +80,48 @@ void RareDestination::protocolDetected(Flow *f) { u_int32_t hash = getDestinationHash(f); if(hash == 0) return; - /* training */ + /* if training */ if (cli_lhost->isTrainingRareDest()) { ndpi_bitmap_set(rare_dest, hash); //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Hash %s added ~ %s", f->getFlowServerInfo(), host_id ); /* check if training has to end */ - if (t_now - cli_lhost->getStartRareDestTraining() >= 300 /* RARE_DEST_DURATION_TRAINING */ ) - { - cli_lhost->addRareDestBitmaps(); + if (t_now - cli_lhost->getStartRareDestTraining() >= 300 /* RARE_DEST_DURATION_TRAINING */ ) { cli_lhost->stopRareDestTraining(); cli_lhost->setLastRareDestTraining(t_now); - //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training Off at %ld ~ %s", t_now, host_id ); } return; } - if ( t_now - cli_lhost->getLastRareDestTraining() > 600 /* RARE_DEST_LAST_TRAINING_GAP */ ) + /* check if training has to restart */ + time_t elapsedFromLastTraining = t_now - cli_lhost->getLastRareDestTraining(); + + if (elapsedFromLastTraining >= 2*600 /* 2*RARE_DEST_LAST_TRAINING_GAP */ ) { + cli_lhost->clearRareDestBitmaps(); + cli_lhost->setStartRareDestTraining(0); + + return; + } + + if ( elapsedFromLastTraining >= 600 /* RARE_DEST_LAST_TRAINING_GAP */ ) { - cli_lhost->mergeRareDestBitmaps(); + cli_lhost->updateRareDestLastBitmap(); + cli_lhost->ndpi_bitmap_clear(rare_dest); cli_lhost->setStartRareDestTraining(0); - // ndpi_bitmap_clear(rare_dest); useless, see mergeRareDestBitmaps - //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Merged Bitmaps %s", host_id ); + return; } /* update */ if (!ndpi_bitmap_isset(rare_dest, hash)){ - ndpi_bitmap_set(rare_dest, hash); // Fire an alarm only once - is_rare_destination = true; + ndpi_bitmap_set(rare_dest, hash); + if (!ndpi_bitmap_isset(rare_dest_last, hash)) { + is_rare_destination = true; + } } - } if (is_rare_destination) { From 73c75ec17611cd09125d38c816eea739b348f72e Mon Sep 17 00:00:00 2001 From: Zocco Giuseppe Date: Mon, 31 Jul 2023 16:47:55 +0200 Subject: [PATCH 32/50] Rimozione puntatore a bitmap in RareDestination.cpp e fix relativi metodi --- include/LocalHost.h | 13 +++++++++++-- src/flow_checks/RareDestination.cpp | 18 +++++++++++------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/include/LocalHost.h b/include/LocalHost.h index 9ed30a0f8cd2..1ec3236c0143 100644 --- a/include/LocalHost.h +++ b/include/LocalHost.h @@ -39,6 +39,7 @@ class LocalHost : public Host { /* RareDestination data implementation*/ ndpi_bitmap *rare_dest; ndpi_bitmap *rare_dest_last; + u_int32_t rare_dest_hash; struct { time_t start; @@ -194,8 +195,8 @@ class LocalHost : public Host { /* RareDest Extension methods */ - inline ndpi_bitmap* getRareDestBitmap() const { return(rare_dest); } - inline ndpi_bitmap* getRareDestLastBitmap() const { return(rare_dest_last); } + //inline ndpi_bitmap* getRareDestBitmap() const { return(rare_dest); } + //inline ndpi_bitmap* getRareDestLastBitmap() const { return(rare_dest_last); } inline time_t getStartRareDestTraining() const { return(rareDestTraining.start); } inline void setStartRareDestTraining(time_t t) { rareDestTraining.start = t; } @@ -203,12 +204,20 @@ class LocalHost : public Host { inline time_t getLastRareDestTraining() const { return(rareDestTraining.last_training); } inline void setLastRareDestTraining(time_t t) { rareDestTraining.last_training = t; } + inline void setRareDestHash(u_int32_t hash) { rare_dest_hash = hash; } + inline bool isTrainingRareDest() const { return(rareDestTraining.training); } inline void startRareDestTraining() { rareDestTraining.training = true; } inline void stopRareDestTraining() { rareDestTraining.training = false; } inline void updateRareDestLastBitmap() { ndpi_bitmap_clear(rare_dest_last); ndpi_bitmap_or(rare_dest_last, rare_dest); } + inline void clearRareDestBitmap() { ndpi_bitmap_clear(rare_dest); } + inline void clearRareDestLastBitmaps() { ndpi_bitmap_clear(rare_dest_last); } inline void clearRareDestBitmaps() { ndpi_bitmap_clear(rare_dest); ndpi_bitmap_clear(rare_dest_last); } + + inline void setRareDestBitmap() { if(rare_dest) ndpi_bitmap_set(rare_dest, hash); } + inline bool isSetRareDestBitmap() const { if(rare_dest) return ndpi_bitmap_isset(rare_dest, hash); return false;} + inline bool isSetRareDestLastBitmap() const { if(rare_dest_last) return ndpi_bitmap_isset(rare_dest_last, hash); return false;} void dumpRareDestToRedis(); bool loadRareDestFromRedis(); diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index 86bf4a0b0766..503ef14845e0 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -62,8 +62,8 @@ void RareDestination::protocolDetected(Flow *f) { if(f->getFlowServerInfo() != NULL && f->get_cli_host()->isLocalHost()) { LocalHost *cli_lhost = (LocalHost*)f->get_cli_host(); - ndpi_bitmap *rare_dest = cli_lhost->getRareDestBitmap(); - ndpi_bitmap *rare_dest_last = cli_lhost->getRareDestLastBitmap(); + //ndpi_bitmap *rare_dest = cli_lhost->getRareDestBitmap(); + //ndpi_bitmap *rare_dest_last = cli_lhost->getRareDestLastBitmap(); /* char hostbuf[64], *host_id; host_id = cli_lhost->get_hostkey(hostbuf, sizeof(hostbuf)); */ @@ -79,10 +79,12 @@ void RareDestination::protocolDetected(Flow *f) { u_int32_t hash = getDestinationHash(f); if(hash == 0) return; + cli_lhost->setRareDestHash(hash); /* if training */ if (cli_lhost->isTrainingRareDest()) { - ndpi_bitmap_set(rare_dest, hash); + cli_lhost->setRareDestBitmap(); + //ndpi_bitmap_set(rare_dest, hash); //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Hash %s added ~ %s", f->getFlowServerInfo(), host_id ); /* check if training has to end */ @@ -108,7 +110,8 @@ void RareDestination::protocolDetected(Flow *f) { if ( elapsedFromLastTraining >= 600 /* RARE_DEST_LAST_TRAINING_GAP */ ) { cli_lhost->updateRareDestLastBitmap(); - cli_lhost->ndpi_bitmap_clear(rare_dest); + cli_lhost->clearRareDestBitmap(); + //cli_lhost->ndpi_bitmap_clear(rare_dest); cli_lhost->setStartRareDestTraining(0); //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Merged Bitmaps %s", host_id ); @@ -116,9 +119,10 @@ void RareDestination::protocolDetected(Flow *f) { } /* update */ - if (!ndpi_bitmap_isset(rare_dest, hash)){ - ndpi_bitmap_set(rare_dest, hash); - if (!ndpi_bitmap_isset(rare_dest_last, hash)) { + if (!cli_lhost->isSetRareDestBitmap()){ + //ndpi_bitmap_set(rare_dest, hash); + cli_lhost->setRareDestBitmap(); + if (!cli_lhost->isSetRareDestLastBitmap()) { is_rare_destination = true; } } From e436dae586a77bf31fcbcc04b92c26d5acc19b97 Mon Sep 17 00:00:00 2001 From: LeoBubi Date: Mon, 31 Jul 2023 21:33:09 +0200 Subject: [PATCH 33/50] Minor fix --- include/LocalHost.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/LocalHost.h b/include/LocalHost.h index 1ec3236c0143..4079e115a04b 100644 --- a/include/LocalHost.h +++ b/include/LocalHost.h @@ -215,9 +215,9 @@ class LocalHost : public Host { inline void clearRareDestLastBitmaps() { ndpi_bitmap_clear(rare_dest_last); } inline void clearRareDestBitmaps() { ndpi_bitmap_clear(rare_dest); ndpi_bitmap_clear(rare_dest_last); } - inline void setRareDestBitmap() { if(rare_dest) ndpi_bitmap_set(rare_dest, hash); } - inline bool isSetRareDestBitmap() const { if(rare_dest) return ndpi_bitmap_isset(rare_dest, hash); return false;} - inline bool isSetRareDestLastBitmap() const { if(rare_dest_last) return ndpi_bitmap_isset(rare_dest_last, hash); return false;} + inline void setRareDestBitmap() { if(rare_dest) ndpi_bitmap_set(rare_dest, rare_dest_hash); } + inline bool isSetRareDestBitmap() const { if(rare_dest) return ndpi_bitmap_isset(rare_dest, rare_dest_hash); return false;} + inline bool isSetRareDestLastBitmap() const { if(rare_dest_last) return ndpi_bitmap_isset(rare_dest_last, rare_dest_hash); return false;} void dumpRareDestToRedis(); bool loadRareDestFromRedis(); From 6cad5773956070e1ecac9fff1da9ffda5e51a420 Mon Sep 17 00:00:00 2001 From: LeoBubi Date: Mon, 31 Jul 2023 21:50:14 +0200 Subject: [PATCH 34/50] Method refactor for performance improvement --- include/LocalHost.h | 2 +- src/flow_checks/RareDestination.cpp | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/include/LocalHost.h b/include/LocalHost.h index 4079e115a04b..b164dd0af9c9 100644 --- a/include/LocalHost.h +++ b/include/LocalHost.h @@ -210,7 +210,7 @@ class LocalHost : public Host { inline void startRareDestTraining() { rareDestTraining.training = true; } inline void stopRareDestTraining() { rareDestTraining.training = false; } - inline void updateRareDestLastBitmap() { ndpi_bitmap_clear(rare_dest_last); ndpi_bitmap_or(rare_dest_last, rare_dest); } + inline void updateRareDestLastBitmap() { ndpi_bitmap_free(rare_dest_last); rare_dest_last = rare_dest; rare_dest = ndpi_bitmap_alloc(); } inline void clearRareDestBitmap() { ndpi_bitmap_clear(rare_dest); } inline void clearRareDestLastBitmaps() { ndpi_bitmap_clear(rare_dest_last); } inline void clearRareDestBitmaps() { ndpi_bitmap_clear(rare_dest); ndpi_bitmap_clear(rare_dest_last); } diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index 503ef14845e0..9ed48e9581d4 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -110,8 +110,6 @@ void RareDestination::protocolDetected(Flow *f) { if ( elapsedFromLastTraining >= 600 /* RARE_DEST_LAST_TRAINING_GAP */ ) { cli_lhost->updateRareDestLastBitmap(); - cli_lhost->clearRareDestBitmap(); - //cli_lhost->ndpi_bitmap_clear(rare_dest); cli_lhost->setStartRareDestTraining(0); //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Merged Bitmaps %s", host_id ); From 4ea53a8dc6fca204a2772dfb0eabce2e90715951 Mon Sep 17 00:00:00 2001 From: Zocco Giuseppe Date: Tue, 1 Aug 2023 00:58:05 +0200 Subject: [PATCH 35/50] Fix metodi set e isSet per Bitmap --- include/LocalHost.h | 9 +++------ src/flow_checks/RareDestination.cpp | 9 ++++----- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/include/LocalHost.h b/include/LocalHost.h index 1ec3236c0143..b9cd4d888d73 100644 --- a/include/LocalHost.h +++ b/include/LocalHost.h @@ -39,7 +39,6 @@ class LocalHost : public Host { /* RareDestination data implementation*/ ndpi_bitmap *rare_dest; ndpi_bitmap *rare_dest_last; - u_int32_t rare_dest_hash; struct { time_t start; @@ -204,8 +203,6 @@ class LocalHost : public Host { inline time_t getLastRareDestTraining() const { return(rareDestTraining.last_training); } inline void setLastRareDestTraining(time_t t) { rareDestTraining.last_training = t; } - inline void setRareDestHash(u_int32_t hash) { rare_dest_hash = hash; } - inline bool isTrainingRareDest() const { return(rareDestTraining.training); } inline void startRareDestTraining() { rareDestTraining.training = true; } inline void stopRareDestTraining() { rareDestTraining.training = false; } @@ -215,9 +212,9 @@ class LocalHost : public Host { inline void clearRareDestLastBitmaps() { ndpi_bitmap_clear(rare_dest_last); } inline void clearRareDestBitmaps() { ndpi_bitmap_clear(rare_dest); ndpi_bitmap_clear(rare_dest_last); } - inline void setRareDestBitmap() { if(rare_dest) ndpi_bitmap_set(rare_dest, hash); } - inline bool isSetRareDestBitmap() const { if(rare_dest) return ndpi_bitmap_isset(rare_dest, hash); return false;} - inline bool isSetRareDestLastBitmap() const { if(rare_dest_last) return ndpi_bitmap_isset(rare_dest_last, hash); return false;} + inline void setRareDestBitmap(u_int32_t hash) { if(rare_dest) ndpi_bitmap_set(rare_dest, hash); } + inline bool isSetRareDestBitmap(u_int32_t hash) const { if(rare_dest) return ndpi_bitmap_isset(rare_dest, hash); return false;} + inline bool isSetRareDestLastBitmap(u_int32_t hash) const { if(rare_dest_last) return ndpi_bitmap_isset(rare_dest_last, hash); return false;} void dumpRareDestToRedis(); bool loadRareDestFromRedis(); diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index 503ef14845e0..0a25243a8fb9 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -79,11 +79,10 @@ void RareDestination::protocolDetected(Flow *f) { u_int32_t hash = getDestinationHash(f); if(hash == 0) return; - cli_lhost->setRareDestHash(hash); /* if training */ if (cli_lhost->isTrainingRareDest()) { - cli_lhost->setRareDestBitmap(); + cli_lhost->setRareDestBitmap(hash); //ndpi_bitmap_set(rare_dest, hash); //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Hash %s added ~ %s", f->getFlowServerInfo(), host_id ); @@ -119,10 +118,10 @@ void RareDestination::protocolDetected(Flow *f) { } /* update */ - if (!cli_lhost->isSetRareDestBitmap()){ + if (!cli_lhost->isSetRareDestBitmap(hash)){ //ndpi_bitmap_set(rare_dest, hash); - cli_lhost->setRareDestBitmap(); - if (!cli_lhost->isSetRareDestLastBitmap()) { + cli_lhost->setRareDestBitmap(hash); + if (!cli_lhost->isSetRareDestLastBitmap(hash)) { is_rare_destination = true; } } From 981b955756cf0e2ef4ae5696fd350fe88435e5e3 Mon Sep 17 00:00:00 2001 From: LeoBubi Date: Tue, 1 Aug 2023 08:19:19 +0200 Subject: [PATCH 36/50] Code fix and method renaming --- include/LocalHost.h | 15 ++++++--------- src/flow_checks/RareDestination.cpp | 8 ++------ 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/include/LocalHost.h b/include/LocalHost.h index 77300adb13c9..feff28fdb945 100644 --- a/include/LocalHost.h +++ b/include/LocalHost.h @@ -193,9 +193,6 @@ class LocalHost : public Host { virtual inline std::unordered_map* getServerPorts(bool isTCP) { return(usedPorts.getServerPorts(isTCP)); }; /* RareDest Extension methods */ - - //inline ndpi_bitmap* getRareDestBitmap() const { return(rare_dest); } - //inline ndpi_bitmap* getRareDestLastBitmap() const { return(rare_dest_last); } inline time_t getStartRareDestTraining() const { return(rareDestTraining.start); } inline void setStartRareDestTraining(time_t t) { rareDestTraining.start = t; } @@ -207,13 +204,13 @@ class LocalHost : public Host { inline void startRareDestTraining() { rareDestTraining.training = true; } inline void stopRareDestTraining() { rareDestTraining.training = false; } - inline void updateRareDestLastBitmap() { ndpi_bitmap_clear(rare_dest_last); ndpi_bitmap_or(rare_dest_last, rare_dest); } - inline void clearRareDestBitmap() { ndpi_bitmap_clear(rare_dest); } - inline void clearRareDestLastBitmaps() { ndpi_bitmap_clear(rare_dest_last); } - inline void updateRareDestLastBitmap() { ndpi_bitmap_free(rare_dest_last); rare_dest_last = rare_dest; rare_dest = ndpi_bitmap_alloc(); } + // inline void clearRareDestBitmap() { ndpi_bitmap_clear(rare_dest); } + // inline void clearRareDestLastBitmap() { ndpi_bitmap_clear(rare_dest_last); } + inline void clearBothRareDestBitmaps() { ndpi_bitmap_clear(rare_dest); ndpi_bitmap_clear(rare_dest_last); } + inline void updateBothRareDestBitmaps() { ndpi_bitmap_free(rare_dest_last); rare_dest_last = rare_dest; rare_dest = ndpi_bitmap_alloc(); } - inline void setRareDestBitmap(u_int32_t hash) { if(rare_dest) ndpi_bitmap_set(rare_dest, hash); } - inline bool isSetRareDestBitmap(u_int32_t hash) const { if(rare_dest) return ndpi_bitmap_isset(rare_dest, hash); return false;} + inline void setRareDestBitmap(u_int32_t hash) { if(rare_dest) ndpi_bitmap_set(rare_dest, hash); } + inline bool isSetRareDestBitmap(u_int32_t hash) const { if(rare_dest) return ndpi_bitmap_isset(rare_dest, hash); return false;} inline bool isSetRareDestLastBitmap(u_int32_t hash) const { if(rare_dest_last) return ndpi_bitmap_isset(rare_dest_last, hash); return false;} void dumpRareDestToRedis(); diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index 3bf18cb2368d..153297cde15c 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -62,8 +62,6 @@ void RareDestination::protocolDetected(Flow *f) { if(f->getFlowServerInfo() != NULL && f->get_cli_host()->isLocalHost()) { LocalHost *cli_lhost = (LocalHost*)f->get_cli_host(); - //ndpi_bitmap *rare_dest = cli_lhost->getRareDestBitmap(); - //ndpi_bitmap *rare_dest_last = cli_lhost->getRareDestLastBitmap(); /* char hostbuf[64], *host_id; host_id = cli_lhost->get_hostkey(hostbuf, sizeof(hostbuf)); */ @@ -83,7 +81,6 @@ void RareDestination::protocolDetected(Flow *f) { /* if training */ if (cli_lhost->isTrainingRareDest()) { cli_lhost->setRareDestBitmap(hash); - //ndpi_bitmap_set(rare_dest, hash); //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Hash %s added ~ %s", f->getFlowServerInfo(), host_id ); /* check if training has to end */ @@ -100,7 +97,7 @@ void RareDestination::protocolDetected(Flow *f) { time_t elapsedFromLastTraining = t_now - cli_lhost->getLastRareDestTraining(); if (elapsedFromLastTraining >= 2*600 /* 2*RARE_DEST_LAST_TRAINING_GAP */ ) { - cli_lhost->clearRareDestBitmaps(); + cli_lhost->clearBothRareDestBitmaps(); cli_lhost->setStartRareDestTraining(0); return; @@ -108,7 +105,7 @@ void RareDestination::protocolDetected(Flow *f) { if ( elapsedFromLastTraining >= 600 /* RARE_DEST_LAST_TRAINING_GAP */ ) { - cli_lhost->updateRareDestLastBitmap(); + cli_lhost->updateBothRareDestBitmaps(); cli_lhost->setStartRareDestTraining(0); //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Merged Bitmaps %s", host_id ); @@ -117,7 +114,6 @@ void RareDestination::protocolDetected(Flow *f) { /* update */ if (!cli_lhost->isSetRareDestBitmap(hash)){ - //ndpi_bitmap_set(rare_dest, hash); cli_lhost->setRareDestBitmap(hash); if (!cli_lhost->isSetRareDestLastBitmap(hash)) { is_rare_destination = true; From 54c149a0dd5a20360bf8bc984981ef9999073860 Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Tue, 5 Sep 2023 09:52:16 +0200 Subject: [PATCH 37/50] removed commented prints --- include/LocalHost.h | 6 ++---- include/ntop_defines.h | 4 ++-- src/LocalHost.cpp | 2 -- src/flow_checks/RareDestination.cpp | 30 +++++++++-------------------- 4 files changed, 13 insertions(+), 29 deletions(-) diff --git a/include/LocalHost.h b/include/LocalHost.h index feff28fdb945..cd46747ed8fc 100644 --- a/include/LocalHost.h +++ b/include/LocalHost.h @@ -204,10 +204,8 @@ class LocalHost : public Host { inline void startRareDestTraining() { rareDestTraining.training = true; } inline void stopRareDestTraining() { rareDestTraining.training = false; } - // inline void clearRareDestBitmap() { ndpi_bitmap_clear(rare_dest); } - // inline void clearRareDestLastBitmap() { ndpi_bitmap_clear(rare_dest_last); } - inline void clearBothRareDestBitmaps() { ndpi_bitmap_clear(rare_dest); ndpi_bitmap_clear(rare_dest_last); } - inline void updateBothRareDestBitmaps() { ndpi_bitmap_free(rare_dest_last); rare_dest_last = rare_dest; rare_dest = ndpi_bitmap_alloc(); } + inline void clearRareDestBitmaps() { ndpi_bitmap_clear(rare_dest); ndpi_bitmap_clear(rare_dest_last); } + inline void updateRareDestBitmaps() { ndpi_bitmap_free(rare_dest_last); rare_dest_last = rare_dest; rare_dest = ndpi_bitmap_alloc(); } inline void setRareDestBitmap(u_int32_t hash) { if(rare_dest) ndpi_bitmap_set(rare_dest, hash); } inline bool isSetRareDestBitmap(u_int32_t hash) const { if(rare_dest) return ndpi_bitmap_isset(rare_dest, hash); return false;} diff --git a/include/ntop_defines.h b/include/ntop_defines.h index 4b606aacb564..a8aff190c8a5 100644 --- a/include/ntop_defines.h +++ b/include/ntop_defines.h @@ -1488,8 +1488,8 @@ extern struct ntopngLuaContext *getUserdata(struct lua_State *vm); /******************************************************************************/ #define HOST_RARE_DEST_SERIALIZED_KEY "ntopng.localhost_rare_dest_fields.%s" -#define RARE_DEST_DURATION_TRAINING 3600 /* seconds ( 1 hour ) */ -#define RARE_DEST_LAST_TRAINING_GAP 7200 /* seconds ( 2 hours ) */ +#define RARE_DEST_DURATION_TRAINING 10800 /* seconds ( 3 hours ) */ +#define RARE_DEST_LAST_TRAINING_GAP 86400 /* seconds ( 1 day ) */ /******************************************************************************/ diff --git a/src/LocalHost.cpp b/src/LocalHost.cpp index a40a847c383e..5cb5026d53ae 100644 --- a/src/LocalHost.cpp +++ b/src/LocalHost.cpp @@ -597,7 +597,6 @@ void LocalHost::dumpRareDestToRedis() { snprintf(param_ser, sizeof(param_ser), "%d", rareDestTraining.training ? 1 : 0); redis->hashSet(key, buf, param_ser); - /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data dumped for %s", get_hostkey(hostbuf, sizeof(hostbuf))); */ } bool LocalHost::loadRareDestFromRedis() { @@ -662,6 +661,5 @@ bool LocalHost::loadRareDestFromRedis() { rare_dest_last = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); free(value); - /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "Rare Dest data loaded for %s", get_hostkey(hostbuf, sizeof(hostbuf))); */ return(true); } diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index 153297cde15c..bf52162a0abb 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -32,21 +32,18 @@ u_int32_t RareDestination::getDestinationHash(Flow *f) { if (f->isLocalToLocal()) { char buf[64]; - + if (!dest->isMulticastHost() && dest->isDHCPHost()) { char *mac = dest->getMac()->print(buf,sizeof(buf)); hash = Utils::hashString(mac); - /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination MAC %u - %s", *hash, mac); */ } - else if (dest->isIPv6() || dest->isIPv4()) { char *ip = dest->get_ip()->print(buf,sizeof(buf)); hash = Utils::hashString(ip); - /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare local destination IPv6/IPv4 %u - %s", *hash, ip); */ } - } - else if (f->isLocalToRemote()){ + } + else if (f->isLocalToRemote()) { hash = Utils::hashString(f->getFlowServerInfo()); } @@ -61,18 +58,14 @@ void RareDestination::protocolDetected(Flow *f) { if(f->getFlowServerInfo() != NULL && f->get_cli_host()->isLocalHost()) { - LocalHost *cli_lhost = (LocalHost*)f->get_cli_host(); + LocalHost *cli_lhost = (LocalHost*)f->get_cli_host(); - /* char hostbuf[64], *host_id; - host_id = cli_lhost->get_hostkey(hostbuf, sizeof(hostbuf)); */ - time_t t_now = time(NULL); /* check if training has to start */ if (!cli_lhost->getStartRareDestTraining()) { cli_lhost->startRareDestTraining(); cli_lhost->setStartRareDestTraining(t_now); - //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training On at %ld ~ %s", t_now, host_id ); } u_int32_t hash = getDestinationHash(f); @@ -81,13 +74,11 @@ void RareDestination::protocolDetected(Flow *f) { /* if training */ if (cli_lhost->isTrainingRareDest()) { cli_lhost->setRareDestBitmap(hash); - //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Hash %s added ~ %s", f->getFlowServerInfo(), host_id ); /* check if training has to end */ - if (t_now - cli_lhost->getStartRareDestTraining() >= 300 /* RARE_DEST_DURATION_TRAINING */ ) { + if (t_now - cli_lhost->getStartRareDestTraining() >= 3600 /* RARE_DEST_DURATION_TRAINING */ ) { cli_lhost->stopRareDestTraining(); cli_lhost->setLastRareDestTraining(t_now); - //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Training Off at %ld ~ %s", t_now, host_id ); } return; @@ -96,19 +87,16 @@ void RareDestination::protocolDetected(Flow *f) { /* check if training has to restart */ time_t elapsedFromLastTraining = t_now - cli_lhost->getLastRareDestTraining(); - if (elapsedFromLastTraining >= 2*600 /* 2*RARE_DEST_LAST_TRAINING_GAP */ ) { - cli_lhost->clearBothRareDestBitmaps(); + if (elapsedFromLastTraining >= 2*RARE_DEST_LAST_TRAINING_GAP ) { + cli_lhost->clearRareDestBitmaps(); cli_lhost->setStartRareDestTraining(0); return; } - if ( elapsedFromLastTraining >= 600 /* RARE_DEST_LAST_TRAINING_GAP */ ) - { - cli_lhost->updateBothRareDestBitmaps(); + if ( elapsedFromLastTraining >= RARE_DEST_LAST_TRAINING_GAP ) { + cli_lhost->updateRareDestBitmaps(); cli_lhost->setStartRareDestTraining(0); - //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Merged Bitmaps %s", host_id ); - return; } From 0ed8b58d44a5fd1c72c69ed24c92ca3a7aed0909 Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Tue, 5 Sep 2023 10:37:35 +0200 Subject: [PATCH 38/50] removed trailing whitespacing after changes --- include/Host.h | 3 +-- src/Host.cpp | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/Host.h b/include/Host.h index b30ee335a090..567b87793f94 100644 --- a/include/Host.h +++ b/include/Host.h @@ -75,7 +75,7 @@ class Host : public GenericHashEntry, } names; char *ssdpLocation; - + /* END Host data: */ /* Counters used by host alerts */ @@ -1000,7 +1000,6 @@ class Host : public GenericHashEntry, ndpi_bitmap_set(tcp_udp_contacted_ports_no_tx, port); } - void resetHostContacts(); }; diff --git a/src/Host.cpp b/src/Host.cpp index cca7e1f8cf0b..69a5f5a127c9 100644 --- a/src/Host.cpp +++ b/src/Host.cpp @@ -332,7 +332,8 @@ void Host::initialize(Mac *_mac, u_int16_t _vlanId, ndpi_hll_init(&incoming_hosts_tcp_udp_port_with_no_tx_hll, 5 /* StdError: 18.4% */); - deferredInitialization(); /* TODO To be called asynchronously for improving performance */ + deferredInitialization(); /* TODO To be called asynchronously for improving + performance */ } /* *************************************** */ From cbd7892dccf82d86d8bcc6589de3f3edc4b7661b Mon Sep 17 00:00:00 2001 From: Leonardo Brugnano Date: Tue, 12 Sep 2023 16:24:39 +0200 Subject: [PATCH 39/50] Hashing code fix --- src/flow_checks/RareDestination.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index bf52162a0abb..303289be2383 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -32,14 +32,15 @@ u_int32_t RareDestination::getDestinationHash(Flow *f) { if (f->isLocalToLocal()) { char buf[64]; + LocalHost *ldest = (LocalHost*)dest; - if (!dest->isMulticastHost() && dest->isDHCPHost()) { - char *mac = dest->getMac()->print(buf,sizeof(buf)); - hash = Utils::hashString(mac); + if (!ldest->isLocalUnicastHost() && dest->isDHCPHost()) { + u_int8_t *mac = dest->getMac()->get_mac(); + hash = Utils::macHash(mac); } else if (dest->isIPv6() || dest->isIPv4()) { - char *ip = dest->get_ip()->print(buf,sizeof(buf)); - hash = Utils::hashString(ip); + IpAddress *ip = dest->get_ip(); + hash = ip->key(); } } From 397cd3e324b843b7d48a85faf10f1d8dc37706e4 Mon Sep 17 00:00:00 2001 From: Leonardo Brugnano Date: Thu, 14 Sep 2023 15:34:03 +0200 Subject: [PATCH 40/50] Background training implementation --- include/ntop_defines.h | 4 +- src/flow_checks/RareDestination.cpp | 75 +++++++++++++++-------------- 2 files changed, 40 insertions(+), 39 deletions(-) diff --git a/include/ntop_defines.h b/include/ntop_defines.h index 14350773f148..cfaed4ca7d76 100644 --- a/include/ntop_defines.h +++ b/include/ntop_defines.h @@ -1506,8 +1506,8 @@ extern struct ntopngLuaContext *getUserdata(struct lua_State *vm); /******************************************************************************/ #define HOST_RARE_DEST_SERIALIZED_KEY "ntopng.localhost_rare_dest_fields.%s" -#define RARE_DEST_DURATION_TRAINING 10800 /* seconds ( 3 hours ) */ -#define RARE_DEST_LAST_TRAINING_GAP 86400 /* seconds ( 1 day ) */ +#define RARE_DEST_TRAINING_DURATION 10800 /* seconds ( 3 hours ) */ +#define RARE_DEST_TRAINING_GAP 86400 /* seconds ( 1 day ) */ /******************************************************************************/ diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index 303289be2383..b99ddca373ef 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -24,14 +24,14 @@ /* ***************************************************** */ -u_int32_t RareDestination::getDestinationHash(Flow *f) { +u_int32_t RareDestination::getDestinationHash(Flow *f, u_int8_t *destType) { u_int32_t hash = 0; Host *dest = f->get_srv_host(); if (f->isLocalToLocal()) { - char buf[64]; + *destType = 0; // local destination LocalHost *ldest = (LocalHost*)dest; if (!ldest->isLocalUnicastHost() && dest->isDHCPHost()) { @@ -45,6 +45,7 @@ u_int32_t RareDestination::getDestinationHash(Flow *f) { } else if (f->isLocalToRemote()) { + *destType = 1; // remote destination hash = Utils::hashString(f->getFlowServerInfo()); } @@ -58,56 +59,56 @@ void RareDestination::protocolDetected(Flow *f) { bool is_rare_destination = false; if(f->getFlowServerInfo() != NULL && f->get_cli_host()->isLocalHost()) { - - LocalHost *cli_lhost = (LocalHost*)f->get_cli_host(); - - time_t t_now = time(NULL); - /* check if training has to start */ - if (!cli_lhost->getStartRareDestTraining()) { - cli_lhost->startRareDestTraining(); - cli_lhost->setStartRareDestTraining(t_now); - } + time_t t_now = time(NULL); + NetworkInterface *iface = f->getInterface(); - u_int32_t hash = getDestinationHash(f); + u_int8_t destType; + u_int32_t hash = getDestinationHash(f, &destType); if(hash == 0) return; - /* if training */ - if (cli_lhost->isTrainingRareDest()) { - cli_lhost->setRareDestBitmap(hash); - /* check if training has to end */ - if (t_now - cli_lhost->getStartRareDestTraining() >= 3600 /* RARE_DEST_DURATION_TRAINING */ ) { - cli_lhost->stopRareDestTraining(); - cli_lhost->setLastRareDestTraining(t_now); - } - - return; - } + /* initial training */ + if (iface->rareDestInitalTraining()) { // check initalTraining -- BEWARE: loadFromRedis -> initialTraining = TRUE - /* check if training has to restart */ - time_t elapsedFromLastTraining = t_now - cli_lhost->getLastRareDestTraining(); + destType == 0 ? iface->setLocalRareDestBitmap(hash) : iface->setRemoteRareDestBitmap(hash); - if (elapsedFromLastTraining >= 2*RARE_DEST_LAST_TRAINING_GAP ) { - cli_lhost->clearRareDestBitmaps(); - cli_lhost->setStartRareDestTraining(0); + if (!iface->rareDestTrainingStartTime()) // check trainingStartTime -- BEWARE: loadFromRedis -> trainingStartTime = 0 + iface->setRareDestTrainingStartTime(t_now); // trainingStartTime = t_now + else if (t_now - iface->rareDestTrainingStartTime() >= RARE_DEST_TRAINING_DURATION) + iface->endRareDestInitialTraining(t_now) // initialTraining = FALSE && trainingEndTime = t_now + return; } - if ( elapsedFromLastTraining >= RARE_DEST_LAST_TRAINING_GAP ) { - cli_lhost->updateRareDestBitmaps(); - cli_lhost->setStartRareDestTraining(0); - return; + /* check if background training has to start */ + if (!iface->rareDestTraining() && // check isTraining -- BEWARE: loadFromRedis -> isTraining = FALSE + t_now - iface->rareDestTrainingEndTime() >= RARE_DEST_TRAINING_GAP) { // check trainingEndTime + iface->startRareDestTraining(t_now); // isTraining = TRUE && trainingStartTime = t_now } - /* update */ - if (!cli_lhost->isSetRareDestBitmap(hash)){ - cli_lhost->setRareDestBitmap(hash); - if (!cli_lhost->isSetRareDestLastBitmap(hash)) { - is_rare_destination = true; + /* background training */ + if (iface->rareDestTraining()) { + destType == 0 ? iface->setLocalRareDestBitmap_BG(hash) : iface->setRemoteRareDestBitmap_BG(hash); + + /* check if background training has to end */ + if (t_now - iface->rareDestTrainingStartTime() >= RARE_DEST_TRAINING_DURATION) { + iface->endRareDestTraining(t_now); // isTraining = FALSE && trainingEndTime = t_now + iface->swapRareDestBitmaps(); // swap(local, localBG), free(localBG), swap(remote, remoteBG), free(remoteBG) } } + + /* update bitmap */ + if (destType == 0 && !iface->isSetLocalrareDestBitmap(hash)) { + is_rare_destination = true; + iface->setLocalRareDestBitmap(hash); + } + + else if (destType == 1 && !iface->isSetRemoteRareDestBitmap(hash)) { + is_rare_destination = true; + iface->setRemoteRareDestBitmap(hash); + } } if (is_rare_destination) { From b7ace8bb8efcab430fb763500a05b3ab2ca2133a Mon Sep 17 00:00:00 2001 From: Zocco Giuseppe Date: Fri, 15 Sep 2023 15:30:57 +0200 Subject: [PATCH 41/50] Swap form LocalHost to NetworkInterface --- include/LocalHost.h | 27 ----------------- include/NetworkInterface.h | 47 +++++++++++++++++++++++++++++ src/flow_checks/RareDestination.cpp | 10 +++--- 3 files changed, 52 insertions(+), 32 deletions(-) diff --git a/include/LocalHost.h b/include/LocalHost.h index 5e357d8aca57..ee65f996fc73 100644 --- a/include/LocalHost.h +++ b/include/LocalHost.h @@ -36,16 +36,6 @@ class LocalHost : public Host { u_int8_t router_mac_set : 1, drop_all_host_traffic : 1, systemHost : 1, _notused : 5; - /* RareDestination data implementation*/ - ndpi_bitmap *rare_dest; - ndpi_bitmap *rare_dest_last; - - struct { - time_t start; - time_t last_training; - bool training; - } rareDestTraining; - /* LocalHost data: update LocalHost::deleteHostData when adding new fields */ char *os_detail; /* END Host data: */ @@ -205,23 +195,6 @@ class LocalHost : public Host { bool isTCP) { return (usedPorts.getServerPorts(isTCP)); }; - - inline time_t getStartRareDestTraining() const { return(rareDestTraining.start); } - inline void setStartRareDestTraining(time_t t) { rareDestTraining.start = t; } - - inline time_t getLastRareDestTraining() const { return(rareDestTraining.last_training); } - inline void setLastRareDestTraining(time_t t) { rareDestTraining.last_training = t; } - - inline bool isTrainingRareDest() const { return(rareDestTraining.training); } - inline void startRareDestTraining() { rareDestTraining.training = true; } - inline void stopRareDestTraining() { rareDestTraining.training = false; } - - inline void clearRareDestBitmaps() { ndpi_bitmap_clear(rare_dest); ndpi_bitmap_clear(rare_dest_last); } - inline void updateRareDestBitmaps() { ndpi_bitmap_free(rare_dest_last); rare_dest_last = rare_dest; rare_dest = ndpi_bitmap_alloc(); } - - inline void setRareDestBitmap(u_int32_t hash) { if(rare_dest) ndpi_bitmap_set(rare_dest, hash); } - inline bool isSetRareDestBitmap(u_int32_t hash) const { if(rare_dest) return ndpi_bitmap_isset(rare_dest, hash); return false;} - inline bool isSetRareDestLastBitmap(u_int32_t hash) const { if(rare_dest_last) return ndpi_bitmap_isset(rare_dest_last, hash); return false;} void dumpRareDestToRedis(); bool loadRareDestFromRedis(); diff --git a/include/NetworkInterface.h b/include/NetworkInterface.h index 0c18ecc14748..fd191084941f 100644 --- a/include/NetworkInterface.h +++ b/include/NetworkInterface.h @@ -106,6 +106,21 @@ class NetworkInterface : public NetworkInterfaceAlertableEntity { u_int32_t local_hosts, remote_hosts; } tot_num_anomalies; AlertsQueue *alertsQueue; + + /* RareDestination data implementation*/ + ndpi_bitmap *rare_dest_local; + ndpi_bitmap *rare_dest_remote; + ndpi_bitmap *rare_dest_local_bg; + ndpi_bitmap *rare_dest_remote_bg; + + struct { + time_t startTime; + bool initialTraining; + bool isTraining; + time_t endTime; + } rareDestTraining; + /***************************************/ + #if defined(NTOPNG_PRO) PeriodicityMap *pMap; ServiceMap *sMap; @@ -1370,6 +1385,38 @@ class NetworkInterface : public NetworkInterfaceAlertableEntity { void *user_data, bool *matched); + /*RareDestination method implementation*/ + + inline void setLocalRareDestBitmap(u_int32_t hash) { if(rare_dest_local) ndpi_bitmap_set(rare_dest_local, hash); } + inline void setRemoteRareDestBitmap(u_int32_t hash) { if(rare_dest_remote) ndpi_bitmap_set(rare_dest_remote, hash); } + inline bool isSetLocalRareDestBitmap(u_int32_t hash) const { if(rare_dest_local) return ndpi_bitmap_isset(rare_dest_local, hash); return false;} + inline bool isSetRemoteRareDestBitmap(u_int32_t hash) const { if(rare_dest_remote) return ndpi_bitmap_isset(rare_dest_remote, hash); return false;} + + inline void setLocalRareDestBitmap_BG(u_int32_t hash) { if(rare_dest_local_bg) ndpi_bitmap_set(rare_dest_local_bg, hash); } + inline void setRemoteRareDestBitmap_BG(u_int32_t hash) { if(rare_dest_remote_bg) ndpi_bitmap_set(rare_dest_remote_bg, hash); } + + inline bool getRareDestInitialTraining() const { return rareDestTraining.initialTraining;} + + inline void endRareDestInitialTraining(time_t t) { rareDestTraining.initialTraining=false; rareDestTraining.endTime=t;} + + inline void startRareDestTraining(time_t t) { rareDestTraining.isTraining = true; rareDestTraining.startTime=t;} + inline void endRareDestTraining(time_t t) { rareDestTraining.isTraining = false; rareDestTraining.endTime=t;} + + inline time_t getRareDestTrainingStartTime() const { return rareDestTraining.startTime;} + inline void setRareDestTrainingStartTime(time_t t) { rareDestTraining.startTime = t;} + + inline time_t getRareDestTrainingEndTime() const { return rareDestTraining.endTime;} + inline void setRareDestTrainingEndTime(time_t t) { rareDestTraining.endTime = t;} + + void swapRareDestBitmaps() { + rare_dest_local = rare_dest_local_bg; + free(rare_dest_local_bg); + rare_dest_remote = rare_dest_remote_bg; + free(rare_dest_remote_bg); + } + + /***************************************/ + #ifdef NTOPNG_PRO static bool compute_client_server_flow_stats(GenericHashEntry *node, void *user_data, bool *matched); diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index b99ddca373ef..8570be1018ef 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -69,14 +69,14 @@ void RareDestination::protocolDetected(Flow *f) { /* initial training */ - if (iface->rareDestInitalTraining()) { // check initalTraining -- BEWARE: loadFromRedis -> initialTraining = TRUE + if (iface->getRareDestInitalTraining()) { // check initalTraining -- BEWARE: loadFromRedis -> initialTraining = TRUE destType == 0 ? iface->setLocalRareDestBitmap(hash) : iface->setRemoteRareDestBitmap(hash); - if (!iface->rareDestTrainingStartTime()) // check trainingStartTime -- BEWARE: loadFromRedis -> trainingStartTime = 0 + if (!iface->getRareDestTrainingStartTime()) // check trainingStartTime -- BEWARE: loadFromRedis -> trainingStartTime = 0 iface->setRareDestTrainingStartTime(t_now); // trainingStartTime = t_now - else if (t_now - iface->rareDestTrainingStartTime() >= RARE_DEST_TRAINING_DURATION) + else if (t_now - iface->getRareDestTrainingStartTime() >= RARE_DEST_TRAINING_DURATION) iface->endRareDestInitialTraining(t_now) // initialTraining = FALSE && trainingEndTime = t_now return; @@ -84,7 +84,7 @@ void RareDestination::protocolDetected(Flow *f) { /* check if background training has to start */ if (!iface->rareDestTraining() && // check isTraining -- BEWARE: loadFromRedis -> isTraining = FALSE - t_now - iface->rareDestTrainingEndTime() >= RARE_DEST_TRAINING_GAP) { // check trainingEndTime + t_now - iface->getRareDestTrainingEndTime() >= RARE_DEST_TRAINING_GAP) { // check trainingEndTime iface->startRareDestTraining(t_now); // isTraining = TRUE && trainingStartTime = t_now } @@ -93,7 +93,7 @@ void RareDestination::protocolDetected(Flow *f) { destType == 0 ? iface->setLocalRareDestBitmap_BG(hash) : iface->setRemoteRareDestBitmap_BG(hash); /* check if background training has to end */ - if (t_now - iface->rareDestTrainingStartTime() >= RARE_DEST_TRAINING_DURATION) { + if (t_now - iface->getRareDestTrainingStartTime() >= RARE_DEST_TRAINING_DURATION) { iface->endRareDestTraining(t_now); // isTraining = FALSE && trainingEndTime = t_now iface->swapRareDestBitmaps(); // swap(local, localBG), free(localBG), swap(remote, remoteBG), free(remoteBG) } From 8a3577fe85bb3a9bab2ee7cd2c8559f694c8f858 Mon Sep 17 00:00:00 2001 From: Zocco Giuseppe Date: Fri, 15 Sep 2023 18:09:26 +0200 Subject: [PATCH 42/50] Fix swap method --- include/NetworkInterface.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/include/NetworkInterface.h b/include/NetworkInterface.h index fd191084941f..2af167853f36 100644 --- a/include/NetworkInterface.h +++ b/include/NetworkInterface.h @@ -1409,10 +1409,15 @@ class NetworkInterface : public NetworkInterfaceAlertableEntity { inline void setRareDestTrainingEndTime(time_t t) { rareDestTraining.endTime = t;} void swapRareDestBitmaps() { + ndpi_bitmap *temp; + temp = rare_dest_local; rare_dest_local = rare_dest_local_bg; - free(rare_dest_local_bg); + rare_dest_local_bg = temp; + temp = rare_dest_remote; rare_dest_remote = rare_dest_remote_bg; - free(rare_dest_remote_bg); + rare_dest_remote_bg = temp; + ndpi_bitmap_clear(rare_dest_local_bg); + ndpi_bitmap_clear(rare_dest_remote_bg); } /***************************************/ From 8c2cd4bb6d21890803a0c666c2b0ab2e28f1db5d Mon Sep 17 00:00:00 2001 From: Leonardo Brugnano Date: Sat, 16 Sep 2023 19:45:22 +0200 Subject: [PATCH 43/50] Added method implementation --- include/NetworkInterface.h | 1 + src/flow_checks/RareDestination.cpp | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/include/NetworkInterface.h b/include/NetworkInterface.h index 2af167853f36..3ad44a9cb218 100644 --- a/include/NetworkInterface.h +++ b/include/NetworkInterface.h @@ -1396,6 +1396,7 @@ class NetworkInterface : public NetworkInterfaceAlertableEntity { inline void setRemoteRareDestBitmap_BG(u_int32_t hash) { if(rare_dest_remote_bg) ndpi_bitmap_set(rare_dest_remote_bg, hash); } inline bool getRareDestInitialTraining() const { return rareDestTraining.initialTraining;} + inline bool getRareDestTraining() const { return rareDestTraining.isTraining; } inline void endRareDestInitialTraining(time_t t) { rareDestTraining.initialTraining=false; rareDestTraining.endTime=t;} diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index 8570be1018ef..eb55f9b33da9 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -83,13 +83,13 @@ void RareDestination::protocolDetected(Flow *f) { } /* check if background training has to start */ - if (!iface->rareDestTraining() && // check isTraining -- BEWARE: loadFromRedis -> isTraining = FALSE + if (!iface->getRareDestTraining() && // check isTraining -- BEWARE: loadFromRedis -> isTraining = FALSE t_now - iface->getRareDestTrainingEndTime() >= RARE_DEST_TRAINING_GAP) { // check trainingEndTime iface->startRareDestTraining(t_now); // isTraining = TRUE && trainingStartTime = t_now } /* background training */ - if (iface->rareDestTraining()) { + if (iface->getRareDestTraining()) { destType == 0 ? iface->setLocalRareDestBitmap_BG(hash) : iface->setRemoteRareDestBitmap_BG(hash); /* check if background training has to end */ From bdd88e247b88cb23ed2f58e33efce7bd53588628 Mon Sep 17 00:00:00 2001 From: Leonardo Brugnano Date: Sun, 17 Sep 2023 12:10:43 +0200 Subject: [PATCH 44/50] Code cleanup --- src/flow_checks/RareDestination.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index eb55f9b33da9..ba8abbb4e10a 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -69,23 +69,23 @@ void RareDestination::protocolDetected(Flow *f) { /* initial training */ - if (iface->getRareDestInitalTraining()) { // check initalTraining -- BEWARE: loadFromRedis -> initialTraining = TRUE + if (iface->getRareDestInitalTraining()) { destType == 0 ? iface->setLocalRareDestBitmap(hash) : iface->setRemoteRareDestBitmap(hash); - if (!iface->getRareDestTrainingStartTime()) // check trainingStartTime -- BEWARE: loadFromRedis -> trainingStartTime = 0 - iface->setRareDestTrainingStartTime(t_now); // trainingStartTime = t_now + if (!iface->getRareDestTrainingStartTime()) + iface->setRareDestTrainingStartTime(t_now); else if (t_now - iface->getRareDestTrainingStartTime() >= RARE_DEST_TRAINING_DURATION) - iface->endRareDestInitialTraining(t_now) // initialTraining = FALSE && trainingEndTime = t_now + iface->endRareDestInitialTraining(t_now); return; } /* check if background training has to start */ - if (!iface->getRareDestTraining() && // check isTraining -- BEWARE: loadFromRedis -> isTraining = FALSE - t_now - iface->getRareDestTrainingEndTime() >= RARE_DEST_TRAINING_GAP) { // check trainingEndTime - iface->startRareDestTraining(t_now); // isTraining = TRUE && trainingStartTime = t_now + if (!iface->getRareDestTraining() && + t_now - iface->getRareDestTrainingEndTime() >= RARE_DEST_TRAINING_GAP) { + iface->startRareDestTraining(t_now); } /* background training */ @@ -94,8 +94,8 @@ void RareDestination::protocolDetected(Flow *f) { /* check if background training has to end */ if (t_now - iface->getRareDestTrainingStartTime() >= RARE_DEST_TRAINING_DURATION) { - iface->endRareDestTraining(t_now); // isTraining = FALSE && trainingEndTime = t_now - iface->swapRareDestBitmaps(); // swap(local, localBG), free(localBG), swap(remote, remoteBG), free(remoteBG) + iface->endRareDestTraining(t_now); + iface->swapRareDestBitmaps(); } } From 8505a07f1fc4ed73e163821fd2dcc52708eec187 Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Sun, 17 Sep 2023 18:18:32 +0200 Subject: [PATCH 45/50] feature&fixes: new implementation of Redis save/load minor bug fixes & typo modular get/set methods for rareDestTraining values LocalHost cleanup --- include/LocalHost.h | 4 - include/NetworkInterface.h | 23 +++-- include/flow_checks/RareDestination.h | 2 +- include/ntop_defines.h | 2 +- src/LocalHost.cpp | 134 -------------------------- src/NetworkInterface.cpp | 128 ++++++++++++++++++++++++ src/flow_checks/RareDestination.cpp | 21 ++-- 7 files changed, 154 insertions(+), 160 deletions(-) diff --git a/include/LocalHost.h b/include/LocalHost.h index ee65f996fc73..cc700f09dd3e 100644 --- a/include/LocalHost.h +++ b/include/LocalHost.h @@ -195,10 +195,6 @@ class LocalHost : public Host { bool isTCP) { return (usedPorts.getServerPorts(isTCP)); }; - - void dumpRareDestToRedis(); - bool loadRareDestFromRedis(); - }; #endif /* _LOCAL_HOST_H_ */ diff --git a/include/NetworkInterface.h b/include/NetworkInterface.h index 3ad44a9cb218..6771aaf6b12a 100644 --- a/include/NetworkInterface.h +++ b/include/NetworkInterface.h @@ -115,9 +115,9 @@ class NetworkInterface : public NetworkInterfaceAlertableEntity { struct { time_t startTime; + time_t endTime; bool initialTraining; bool isTraining; - time_t endTime; } rareDestTraining; /***************************************/ @@ -448,6 +448,15 @@ class NetworkInterface : public NetworkInterfaceAlertableEntity { void build_protocol_flow_stats_lua_rsp(lua_State *vm, AggregatedFlowsStats *fs, u_int32_t size, u_int *num); + void saveRareDestToRedis(); + bool loadRareDestFromRedis(); + + void setRareDestStructRedisField(const char *key, const char *field, u_int64_t value); + bool getRareDestStructRedisField(const char *key, const char *field, u_int64_t *value); + + void setRareDestBitmapRedisField(const char *key, const char *field, ndpi_bitmap *bitmap); + bool getRareDestBitmapRedisField(const char *key, const char *field, ndpi_bitmap *bitmap); + public: /** * @brief A Constructor @@ -1410,15 +1419,13 @@ class NetworkInterface : public NetworkInterfaceAlertableEntity { inline void setRareDestTrainingEndTime(time_t t) { rareDestTraining.endTime = t;} void swapRareDestBitmaps() { - ndpi_bitmap *temp; - temp = rare_dest_local; + ndpi_bitmap_free(rare_dest_local); rare_dest_local = rare_dest_local_bg; - rare_dest_local_bg = temp; - temp = rare_dest_remote; + rare_dest_local_bg = ndpi_bitmap_alloc(); + + ndpi_bitmap_free(rare_dest_remote); rare_dest_remote = rare_dest_remote_bg; - rare_dest_remote_bg = temp; - ndpi_bitmap_clear(rare_dest_local_bg); - ndpi_bitmap_clear(rare_dest_remote_bg); + rare_dest_remote_bg = ndpi_bitmap_alloc(); } /***************************************/ diff --git a/include/flow_checks/RareDestination.h b/include/flow_checks/RareDestination.h index 77e04bd0ff6d..c63798b8d3ef 100644 --- a/include/flow_checks/RareDestination.h +++ b/include/flow_checks/RareDestination.h @@ -26,7 +26,7 @@ class RareDestination : public FlowCheck { private: - u_int32_t getDestinationHash(Flow *f); + u_int32_t getDestinationHash(Flow *f, u_int8_t *destType); public: RareDestination() diff --git a/include/ntop_defines.h b/include/ntop_defines.h index cfaed4ca7d76..a1404345f00c 100644 --- a/include/ntop_defines.h +++ b/include/ntop_defines.h @@ -1505,7 +1505,7 @@ extern struct ntopngLuaContext *getUserdata(struct lua_State *vm); #define OFFLINE_LOCAL_HOSTS_KEY "ntopng.hosts.offline.ifid_%d" /******************************************************************************/ -#define HOST_RARE_DEST_SERIALIZED_KEY "ntopng.localhost_rare_dest_fields.%s" +#define IFACE_RARE_DEST_SERIALIZED_KEY "ntopng.iface_rare_dest_fields.ifid_%d" #define RARE_DEST_TRAINING_DURATION 10800 /* seconds ( 3 hours ) */ #define RARE_DEST_TRAINING_GAP 86400 /* seconds ( 1 day ) */ diff --git a/src/LocalHost.cpp b/src/LocalHost.cpp index 1d40e629d8aa..8b30de7c4f2b 100644 --- a/src/LocalHost.cpp +++ b/src/LocalHost.cpp @@ -48,11 +48,6 @@ LocalHost::LocalHost(NetworkInterface *_iface, char *ipAddress, LocalHost::~LocalHost() { addInactiveData(); if (initial_ts_point) delete (initial_ts_point); - - dumpRareDestToRedis(); - if(rare_dest) ndpi_bitmap_free(rare_dest); - if(rare_dest_last) ndpi_bitmap_free(rare_dest_last); - freeLocalHostData(); } @@ -151,15 +146,6 @@ void LocalHost::initialize() { #ifdef HAVE_NEDGE drop_all_host_traffic = 0; #endif - - if( !loadRareDestFromRedis() ){ - rareDestTraining.start = 0; - rareDestTraining.last_training = 0; - rareDestTraining.training = false; - rare_dest = ndpi_bitmap_alloc(); - rare_dest_last = ndpi_bitmap_alloc(); - } - } /* *************************************** */ @@ -625,123 +611,3 @@ void LocalHost::setRxOnlyHost(bool set_it) { } } } - -/* *************************************** */ - -void LocalHost::dumpRareDestToRedis() { - char key[CONST_MAX_LEN_REDIS_KEY], buf[32], param_ser[32], *value; - Redis *redis = ntop->getRedis(); - size_t size; - - if((!redis) || (!rare_dest) || (!rare_dest_last)) return; - - char hostbuf[64]; - snprintf(key, sizeof(key), HOST_RARE_DEST_SERIALIZED_KEY, get_hostkey(hostbuf, sizeof(hostbuf))); - - snprintf(buf, sizeof(buf), "rare_dest"); - size = ndpi_bitmap_serialize(rare_dest, &value); - if (value) { - char *encoded_bmap = Utils::base64_encode((unsigned char *)value, size); - size = strlen(encoded_bmap); - redis->hashSet(key, buf, encoded_bmap); - free(encoded_bmap); - free(value); - } - - snprintf(buf, sizeof(buf), "rare_dest_len"); - snprintf(param_ser, sizeof(param_ser), "%lu", size); - redis->hashSet(key, buf, param_ser); - - snprintf(buf, sizeof(buf), "rare_dest_last"); - size = ndpi_bitmap_serialize(rare_dest_last, &value); - - if (value) { - char *encoded_bmap = Utils::base64_encode((unsigned char *)value, size); - size = strlen(encoded_bmap); - redis->hashSet(key, buf, encoded_bmap); - free(encoded_bmap); - free(value); - } - - snprintf(buf, sizeof(buf), "rare_dest_last_len"); - snprintf(param_ser, sizeof(param_ser), "%lu", size); - redis->hashSet(key, buf, param_ser); - - snprintf(buf, sizeof(buf), "t_start_training"); - snprintf(param_ser, sizeof(param_ser), "%ld", rareDestTraining.start); - redis->hashSet(key, buf, param_ser); - - snprintf(buf, sizeof(buf), "last_training"); - snprintf(param_ser, sizeof(param_ser), "%ld", rareDestTraining.last_training); - redis->hashSet(key, buf, param_ser); - - snprintf(buf, sizeof(buf), "training"); - snprintf(param_ser, sizeof(param_ser), "%d", rareDestTraining.training ? 1 : 0); - redis->hashSet(key, buf, param_ser); - -} - -bool LocalHost::loadRareDestFromRedis() { - char key[CONST_MAX_LEN_REDIS_KEY], buf[32], param_str[32], *value; - Redis *redis = ntop->getRedis(); - size_t size; - - if((!redis)) return(false); - - char hostbuf[64]; - snprintf(key, sizeof(key), HOST_RARE_DEST_SERIALIZED_KEY, get_hostkey(hostbuf, sizeof(hostbuf))); - - snprintf(buf, sizeof(buf), "t_start_training"); - if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); - rareDestTraining.start = atol(param_str); - - snprintf(buf, sizeof(buf), "last_training"); - if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); - rareDestTraining.last_training = atol(param_str); - - snprintf(buf, sizeof(buf), "training"); - if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); - rareDestTraining.training = atoi(param_str) ? true : false; - - snprintf(buf, sizeof(buf), "rare_dest_len"); - if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) return(false); - size = (size_t)strtoul(param_str, NULL, 10); - - if((value = (char *) malloc(size)) == NULL) { - ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to allocate memory to deserialize %s", key); - return(false); - } - - snprintf(buf, sizeof(buf), "rare_dest"); - if(redis->hashGet(key, buf, value, size) != 0) { - free(value); - return(false); - } - - rare_dest = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); - free(value); - - snprintf(buf, sizeof(buf), "rare_dest_last_len"); - if( redis->hashGet(key, buf, param_str, sizeof(param_str)) != 0 ) { - ndpi_bitmap_free(rare_dest); - return(false); - } - size = (size_t)strtoul(param_str, NULL, 10); - - if((value = (char *) malloc(size)) == NULL) { - ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to allocate memory to deserialize %s", key); - ndpi_bitmap_free(rare_dest); - return(false); - } - - snprintf(buf, sizeof(buf), "rare_dest_last"); - if(redis->hashGet(key, buf, value, size) != 0) { - free(value); - ndpi_bitmap_free(rare_dest); - return(false); - } - rare_dest_last = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)value).c_str()); - free(value); - - return(true); -} diff --git a/src/NetworkInterface.cpp b/src/NetworkInterface.cpp index 7dd3be58a21f..a34f4332bc29 100644 --- a/src/NetworkInterface.cpp +++ b/src/NetworkInterface.cpp @@ -375,6 +375,19 @@ void NetworkInterface::init(const char *interface_name) { customFlowLuaScript_end = NULL; customHostLuaScript = NULL; + if( !loadRareDestFromRedis() || (( time(NULL) - rareDestTraining.endTime ) > (2 * RARE_DEST_TRAINING_DURATION)) ){ + rareDestTraining.startTime = 0; + rareDestTraining.endTime = 0; + rareDestTraining.isTraining = false; + + rareDestTraining.initialTraining = true; + + rare_dest_local = ndpi_bitmap_alloc(); + rare_dest_local_bg = ndpi_bitmap_alloc(); + rare_dest_remote = ndpi_bitmap_alloc(); + rare_dest_remote_bg = ndpi_bitmap_alloc(); + } + INTERFACE_PROFILING_INIT(); } @@ -1022,6 +1035,13 @@ NetworkInterface::~NetworkInterface() { cleanShadownDPI(); if (smart_recording_instance_name) free(smart_recording_instance_name); + + saveRareDestToRedis(); + if(rare_dest_local) ndpi_bitmap_free(rare_dest_local); + if(rare_dest_local_bg) ndpi_bitmap_free(rare_dest_local_bg); + if(rare_dest_local) ndpi_bitmap_free(rare_dest_local); + if(rare_dest_remote) ndpi_bitmap_free(rare_dest_remote); + if(rare_dest_remote_bg) ndpi_bitmap_free(rare_dest_remote_bg); } /* **************************************************** */ @@ -12365,3 +12385,111 @@ void NetworkInterface::getActiveMacs(lua_State *vm) { walker(&begin_slot, true /* walk_all */, walker_macs, active_mac_search_walker, (void *)&s); } + +/* *************************************** */ + +void NetworkInterface::setRareDestStructRedisField(const char *key, const char *field, u_int64_t value){ + char buf[32]; + snprintf(buf, sizeof(buf), "%lu", value); + ntop->getRedis()->hashSet(key, field, buf); +} + +void NetworkInterface::setRareDestBitmapRedisField(const char *key, const char *field, ndpi_bitmap *bitmap){ + char *value; + size_t size; + + size = ndpi_bitmap_serialize(bitmap, &value); + char *encoded_bmap = value ? Utils::base64_encode((unsigned char *)value, size) : NULL; + + if (encoded_bmap != NULL) { + size = strlen(encoded_bmap); + ntop->getRedis()->hashSet(key, field, encoded_bmap); + + char len_field[32]; + snprintf(len_field, sizeof(key), "%s_len", field); + setRareDestStructRedisField(key, len_field, (u_int64_t)size); + + free(encoded_bmap); + free(value); + } +} + +void NetworkInterface::saveRareDestToRedis() { + char key[CONST_MAX_LEN_REDIS_KEY]; + + if((!ntop->getRedis()) || (!rare_dest_local) || (!rare_dest_remote)) return; + + snprintf(key, sizeof(key), IFACE_RARE_DEST_SERIALIZED_KEY, get_id()); + + setRareDestBitmapRedisField(key, "rare_dest_local", rare_dest_local); + setRareDestBitmapRedisField(key, "rare_dest_remote", rare_dest_remote); + + setRareDestStructRedisField(key, "startTime", (u_int64_t)rareDestTraining.startTime); + setRareDestStructRedisField(key, "endTime", (u_int64_t)rareDestTraining.endTime); + setRareDestStructRedisField(key, "isTraining", (u_int64_t)(rareDestTraining.isTraining ? 1 : 0)); + setRareDestStructRedisField(key, "initialTraining", (u_int64_t)(rareDestTraining.initialTraining ? 1 : 0)); +} + +bool NetworkInterface::getRareDestStructRedisField(const char *key, const char *field, u_int64_t *value){ + char buf[32]; + if( ntop->getRedis()->hashGet(key, field, buf, sizeof(buf)) != 0 ) return(false); + *value = strtoul(buf, NULL, 10); + return(true); +} + +bool NetworkInterface::getRareDestBitmapRedisField(const char *key, const char *field, ndpi_bitmap *bitmap){ + char *bitmap_field_val; + u_int64_t value; + size_t size; + + char len_field[32]; + snprintf(len_field, sizeof(key), "%s_len", field); + + if (!getRareDestStructRedisField(key, len_field, &value)) return(false); + size = (size_t)(value); + + if((bitmap_field_val = (char *) malloc(size)) == NULL) { + ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to allocate memory to deserialize %s", key); + return(false); + } + + if(ntop->getRedis()->hashGet(key, field, bitmap_field_val, size) != 0) { + free(bitmap_field_val); + return(false); + } + + bitmap = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)bitmap_field_val).c_str()); + free(bitmap_field_val); + + return(true); +} + + +bool NetworkInterface::loadRareDestFromRedis() { + char key[CONST_MAX_LEN_REDIS_KEY], *bitmap_field_val; + u_int64_t value; + + if((!ntop->getRedis())) return(false); + + snprintf(key, sizeof(key), IFACE_RARE_DEST_SERIALIZED_KEY, get_id()); + + if (!getRareDestStructRedisField(key, "startTime", &value)) return(false); + rareDestTraining.startTime = (time_t)(value); + + if (!getRareDestStructRedisField(key, "endTime", &value)) return(false); + rareDestTraining.endTime = (time_t)(value); + + if (!getRareDestStructRedisField(key, "isTraining", &value)) return(false); + rareDestTraining.isTraining = value ? true : false; + + if (!getRareDestStructRedisField(key, "initialTraining", &value)) return(false); + rareDestTraining.initialTraining = value ? true : false; + + if (!getRareDestBitmapRedisField(key, "rare_dest_local", rare_dest_local)) return(false); + if (!getRareDestBitmapRedisField(key, "rare_dest_local_bg", rare_dest_local_bg)) return(false); + + if (!getRareDestBitmapRedisField(key, "rare_dest_remote", rare_dest_remote)) return(false); + if (!getRareDestBitmapRedisField(key, "rare_dest_remote_bg", rare_dest_remote_bg)) return(false); + + return(true); +} diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index ba8abbb4e10a..e6cb20b662aa 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -27,20 +27,18 @@ u_int32_t RareDestination::getDestinationHash(Flow *f, u_int8_t *destType) { u_int32_t hash = 0; - Host *dest = f->get_srv_host(); if (f->isLocalToLocal()) { *destType = 0; // local destination - LocalHost *ldest = (LocalHost*)dest; + LocalHost *ldest = (LocalHost*)f->get_srv_host(); - if (!ldest->isLocalUnicastHost() && dest->isDHCPHost()) { - u_int8_t *mac = dest->getMac()->get_mac(); - hash = Utils::macHash(mac); + if (!ldest->isLocalUnicastHost() && ldest->isDHCPHost()) { + Mac *mac = ldest->getMac(); + hash = mac ? mac->key() : 0; } - else if (dest->isIPv6() || dest->isIPv4()) { - IpAddress *ip = dest->get_ip(); - hash = ip->key(); + else if (ldest->isIPv6() || ldest->isIPv4()) { + hash = ldest->get_ip()->key(); } } @@ -58,7 +56,7 @@ void RareDestination::protocolDetected(Flow *f) { bool is_rare_destination = false; - if(f->getFlowServerInfo() != NULL && f->get_cli_host()->isLocalHost()) { + if(f->get_cli_host()->isLocalHost() && (f->getFlowServerInfo() != NULL || !f->get_srv_ip_addr()->isEmpty()) ) { time_t t_now = time(NULL); NetworkInterface *iface = f->getInterface(); @@ -67,9 +65,8 @@ void RareDestination::protocolDetected(Flow *f) { u_int32_t hash = getDestinationHash(f, &destType); if(hash == 0) return; - /* initial training */ - if (iface->getRareDestInitalTraining()) { + if (iface->getRareDestInitialTraining()) { destType == 0 ? iface->setLocalRareDestBitmap(hash) : iface->setRemoteRareDestBitmap(hash); @@ -100,7 +97,7 @@ void RareDestination::protocolDetected(Flow *f) { } /* update bitmap */ - if (destType == 0 && !iface->isSetLocalrareDestBitmap(hash)) { + if (destType == 0 && !iface->isSetLocalRareDestBitmap(hash)) { is_rare_destination = true; iface->setLocalRareDestBitmap(hash); } From 6dca7faf9f3e423776e31fffa971a5023554543a Mon Sep 17 00:00:00 2001 From: Leonardo Brugnano Date: Sun, 17 Sep 2023 22:23:49 +0200 Subject: [PATCH 46/50] Periodic background training w/o gaps implementation --- include/NetworkInterface.h | 30 ++++++++++------------------- include/ntop_defines.h | 4 ++-- src/NetworkInterface.cpp | 25 ++++++++---------------- src/flow_checks/RareDestination.cpp | 30 +++++++++++------------------ 4 files changed, 31 insertions(+), 58 deletions(-) diff --git a/include/NetworkInterface.h b/include/NetworkInterface.h index 6771aaf6b12a..17e174c85ef8 100644 --- a/include/NetworkInterface.h +++ b/include/NetworkInterface.h @@ -114,10 +114,8 @@ class NetworkInterface : public NetworkInterfaceAlertableEntity { ndpi_bitmap *rare_dest_remote_bg; struct { - time_t startTime; - time_t endTime; bool initialTraining; - bool isTraining; + time_t checkPoint; } rareDestTraining; /***************************************/ @@ -1396,27 +1394,19 @@ class NetworkInterface : public NetworkInterfaceAlertableEntity { /*RareDestination method implementation*/ - inline void setLocalRareDestBitmap(u_int32_t hash) { if(rare_dest_local) ndpi_bitmap_set(rare_dest_local, hash); } - inline void setRemoteRareDestBitmap(u_int32_t hash) { if(rare_dest_remote) ndpi_bitmap_set(rare_dest_remote, hash); } - inline bool isSetLocalRareDestBitmap(u_int32_t hash) const { if(rare_dest_local) return ndpi_bitmap_isset(rare_dest_local, hash); return false;} - inline bool isSetRemoteRareDestBitmap(u_int32_t hash) const { if(rare_dest_remote) return ndpi_bitmap_isset(rare_dest_remote, hash); return false;} + inline void setLocalRareDestBitmap(u_int32_t hash) { ndpi_bitmap_set(rare_dest_local, hash); } + inline void setRemoteRareDestBitmap(u_int32_t hash) { ndpi_bitmap_set(rare_dest_remote, hash); } + inline bool isSetLocalRareDestBitmap(u_int32_t hash) const { return ndpi_bitmap_isset(rare_dest_local, hash); } + inline bool isSetRemoteRareDestBitmap(u_int32_t hash) const { return ndpi_bitmap_isset(rare_dest_remote, hash); } - inline void setLocalRareDestBitmap_BG(u_int32_t hash) { if(rare_dest_local_bg) ndpi_bitmap_set(rare_dest_local_bg, hash); } - inline void setRemoteRareDestBitmap_BG(u_int32_t hash) { if(rare_dest_remote_bg) ndpi_bitmap_set(rare_dest_remote_bg, hash); } + inline void setLocalRareDestBitmap_BG(u_int32_t hash) { ndpi_bitmap_set(rare_dest_local_bg, hash); } + inline void setRemoteRareDestBitmap_BG(u_int32_t hash) { ndpi_bitmap_set(rare_dest_remote_bg, hash); } inline bool getRareDestInitialTraining() const { return rareDestTraining.initialTraining;} - inline bool getRareDestTraining() const { return rareDestTraining.isTraining; } - - inline void endRareDestInitialTraining(time_t t) { rareDestTraining.initialTraining=false; rareDestTraining.endTime=t;} - - inline void startRareDestTraining(time_t t) { rareDestTraining.isTraining = true; rareDestTraining.startTime=t;} - inline void endRareDestTraining(time_t t) { rareDestTraining.isTraining = false; rareDestTraining.endTime=t;} + inline void endRareDestInitialTraining(time_t t) { rareDestTraining.initialTraining = false; rareDestTraining.checkPoint = t; } - inline time_t getRareDestTrainingStartTime() const { return rareDestTraining.startTime;} - inline void setRareDestTrainingStartTime(time_t t) { rareDestTraining.startTime = t;} - - inline time_t getRareDestTrainingEndTime() const { return rareDestTraining.endTime;} - inline void setRareDestTrainingEndTime(time_t t) { rareDestTraining.endTime = t;} + inline time_t getRareDestTrainingCheckPoint() const { return rareDestTraining.checkPoint; } + inline void setRareDestTrainingCheckPoint(time_t t) { rareDestTraining.checkPoint = t; } void swapRareDestBitmaps() { ndpi_bitmap_free(rare_dest_local); diff --git a/include/ntop_defines.h b/include/ntop_defines.h index a1404345f00c..d8062c9bb919 100644 --- a/include/ntop_defines.h +++ b/include/ntop_defines.h @@ -1506,8 +1506,8 @@ extern struct ntopngLuaContext *getUserdata(struct lua_State *vm); /******************************************************************************/ #define IFACE_RARE_DEST_SERIALIZED_KEY "ntopng.iface_rare_dest_fields.ifid_%d" -#define RARE_DEST_TRAINING_DURATION 10800 /* seconds ( 3 hours ) */ -#define RARE_DEST_TRAINING_GAP 86400 /* seconds ( 1 day ) */ +#define RARE_DEST_INITIAL_TRAINING_DURATION 86400 /* seconds ( 1 day ) */ +#define RARE_DEST_BACKGROUND_TRAINING_DURATION 604800 /* seconds ( 1 week ) */ /******************************************************************************/ diff --git a/src/NetworkInterface.cpp b/src/NetworkInterface.cpp index a34f4332bc29..653005bfe538 100644 --- a/src/NetworkInterface.cpp +++ b/src/NetworkInterface.cpp @@ -375,11 +375,8 @@ void NetworkInterface::init(const char *interface_name) { customFlowLuaScript_end = NULL; customHostLuaScript = NULL; - if( !loadRareDestFromRedis() || (( time(NULL) - rareDestTraining.endTime ) > (2 * RARE_DEST_TRAINING_DURATION)) ){ - rareDestTraining.startTime = 0; - rareDestTraining.endTime = 0; - rareDestTraining.isTraining = false; - + if( !loadRareDestFromRedis() || (( time(NULL) - rareDestTraining.checkPoint ) > (2 * RARE_DEST_BACKGROUND_TRAINING_DURATION)) ){ + rareDestTraining.checkPoint = 0; rareDestTraining.initialTraining = true; rare_dest_local = ndpi_bitmap_alloc(); @@ -1039,7 +1036,6 @@ NetworkInterface::~NetworkInterface() { saveRareDestToRedis(); if(rare_dest_local) ndpi_bitmap_free(rare_dest_local); if(rare_dest_local_bg) ndpi_bitmap_free(rare_dest_local_bg); - if(rare_dest_local) ndpi_bitmap_free(rare_dest_local); if(rare_dest_remote) ndpi_bitmap_free(rare_dest_remote); if(rare_dest_remote_bg) ndpi_bitmap_free(rare_dest_remote_bg); } @@ -12422,11 +12418,12 @@ void NetworkInterface::saveRareDestToRedis() { snprintf(key, sizeof(key), IFACE_RARE_DEST_SERIALIZED_KEY, get_id()); setRareDestBitmapRedisField(key, "rare_dest_local", rare_dest_local); + setRareDestBitmapRedisField(key, "rare_dest_local_bg", rare_dest_local_bg); + setRareDestBitmapRedisField(key, "rare_dest_remote", rare_dest_remote); + setRareDestBitmapRedisField(key, "rare_dest_remote_bg", rare_dest_remote_bg); - setRareDestStructRedisField(key, "startTime", (u_int64_t)rareDestTraining.startTime); - setRareDestStructRedisField(key, "endTime", (u_int64_t)rareDestTraining.endTime); - setRareDestStructRedisField(key, "isTraining", (u_int64_t)(rareDestTraining.isTraining ? 1 : 0)); + setRareDestStructRedisField(key, "checkPoint", (u_int64_t)rareDestTraining.startTime); setRareDestStructRedisField(key, "initialTraining", (u_int64_t)(rareDestTraining.initialTraining ? 1 : 0)); } @@ -12473,14 +12470,8 @@ bool NetworkInterface::loadRareDestFromRedis() { snprintf(key, sizeof(key), IFACE_RARE_DEST_SERIALIZED_KEY, get_id()); - if (!getRareDestStructRedisField(key, "startTime", &value)) return(false); - rareDestTraining.startTime = (time_t)(value); - - if (!getRareDestStructRedisField(key, "endTime", &value)) return(false); - rareDestTraining.endTime = (time_t)(value); - - if (!getRareDestStructRedisField(key, "isTraining", &value)) return(false); - rareDestTraining.isTraining = value ? true : false; + if (!getRareDestStructRedisField(key, "checkPoint", &value)) return(false); + rareDestTraining.checkPoint = (time_t)(value); if (!getRareDestStructRedisField(key, "initialTraining", &value)) return(false); rareDestTraining.initialTraining = value ? true : false; diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index e6cb20b662aa..708a68a8b906 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -70,32 +70,18 @@ void RareDestination::protocolDetected(Flow *f) { destType == 0 ? iface->setLocalRareDestBitmap(hash) : iface->setRemoteRareDestBitmap(hash); - if (!iface->getRareDestTrainingStartTime()) - iface->setRareDestTrainingStartTime(t_now); + if (!iface->getRareDestTrainingCheckPoint()) + iface->setRareDestTrainingCheckPoint(t_now); - else if (t_now - iface->getRareDestTrainingStartTime() >= RARE_DEST_TRAINING_DURATION) + else if (t_now - iface->getRareDestTrainingCheckPoint() >= RARE_DEST_INITIAL_TRAINING_DURATION) iface->endRareDestInitialTraining(t_now); return; } - /* check if background training has to start */ - if (!iface->getRareDestTraining() && - t_now - iface->getRareDestTrainingEndTime() >= RARE_DEST_TRAINING_GAP) { - iface->startRareDestTraining(t_now); - } - /* background training */ - if (iface->getRareDestTraining()) { - destType == 0 ? iface->setLocalRareDestBitmap_BG(hash) : iface->setRemoteRareDestBitmap_BG(hash); - - /* check if background training has to end */ - if (t_now - iface->getRareDestTrainingStartTime() >= RARE_DEST_TRAINING_DURATION) { - iface->endRareDestTraining(t_now); - iface->swapRareDestBitmaps(); - } - } - + destType == 0 ? iface->setLocalRareDestBitmap_BG(hash) : iface->setRemoteRareDestBitmap_BG(hash); + /* update bitmap */ if (destType == 0 && !iface->isSetLocalRareDestBitmap(hash)) { is_rare_destination = true; @@ -106,6 +92,12 @@ void RareDestination::protocolDetected(Flow *f) { is_rare_destination = true; iface->setRemoteRareDestBitmap(hash); } + + /* check if background training has to restart */ + if (t_now - iface->getRareDestTrainingCheckPoint() >= RARE_DEST_BACKGROUND_TRAINING_DURATION) { + iface->setRareDestTrainingCheckPoint(t_now); + iface->swapRareDestBitmaps(); + } } if (is_rare_destination) { From fdf6266ce139112f0b7e763fbcccf0c516a37cb8 Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Sun, 17 Sep 2023 22:44:36 +0200 Subject: [PATCH 47/50] Minor Bug Fixes --- src/NetworkInterface.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/NetworkInterface.cpp b/src/NetworkInterface.cpp index 653005bfe538..657390b14d31 100644 --- a/src/NetworkInterface.cpp +++ b/src/NetworkInterface.cpp @@ -12402,7 +12402,7 @@ void NetworkInterface::setRareDestBitmapRedisField(const char *key, const char * ntop->getRedis()->hashSet(key, field, encoded_bmap); char len_field[32]; - snprintf(len_field, sizeof(key), "%s_len", field); + snprintf(len_field, sizeof(len_field), "%s_len", field); setRareDestStructRedisField(key, len_field, (u_int64_t)size); free(encoded_bmap); @@ -12440,7 +12440,7 @@ bool NetworkInterface::getRareDestBitmapRedisField(const char *key, const char * size_t size; char len_field[32]; - snprintf(len_field, sizeof(key), "%s_len", field); + snprintf(len_field, sizeof(len_field), "%s_len", field); if (!getRareDestStructRedisField(key, len_field, &value)) return(false); size = (size_t)(value); @@ -12463,7 +12463,7 @@ bool NetworkInterface::getRareDestBitmapRedisField(const char *key, const char * bool NetworkInterface::loadRareDestFromRedis() { - char key[CONST_MAX_LEN_REDIS_KEY], *bitmap_field_val; + char key[CONST_MAX_LEN_REDIS_KEY]; u_int64_t value; if((!ntop->getRedis())) return(false); From 7f08d019a4e2912440ff31c2833c947d0379122a Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Sun, 17 Sep 2023 23:01:30 +0200 Subject: [PATCH 48/50] Fixed Typos --- src/NetworkInterface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NetworkInterface.cpp b/src/NetworkInterface.cpp index 657390b14d31..adcebd2b1f64 100644 --- a/src/NetworkInterface.cpp +++ b/src/NetworkInterface.cpp @@ -12423,7 +12423,7 @@ void NetworkInterface::saveRareDestToRedis() { setRareDestBitmapRedisField(key, "rare_dest_remote", rare_dest_remote); setRareDestBitmapRedisField(key, "rare_dest_remote_bg", rare_dest_remote_bg); - setRareDestStructRedisField(key, "checkPoint", (u_int64_t)rareDestTraining.startTime); + setRareDestStructRedisField(key, "checkPoint", (u_int64_t)rareDestTraining.checkPoint); setRareDestStructRedisField(key, "initialTraining", (u_int64_t)(rareDestTraining.initialTraining ? 1 : 0)); } From c58b78284d8e4e4a3521e7d0a8d8d0cfbb2a647b Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Sun, 17 Sep 2023 23:52:57 +0200 Subject: [PATCH 49/50] loadRareDestFromRedis: deallocating chain in case of failre --- src/NetworkInterface.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/NetworkInterface.cpp b/src/NetworkInterface.cpp index adcebd2b1f64..3c4a05f401ae 100644 --- a/src/NetworkInterface.cpp +++ b/src/NetworkInterface.cpp @@ -12461,7 +12461,6 @@ bool NetworkInterface::getRareDestBitmapRedisField(const char *key, const char * return(true); } - bool NetworkInterface::loadRareDestFromRedis() { char key[CONST_MAX_LEN_REDIS_KEY]; u_int64_t value; @@ -12477,10 +12476,22 @@ bool NetworkInterface::loadRareDestFromRedis() { rareDestTraining.initialTraining = value ? true : false; if (!getRareDestBitmapRedisField(key, "rare_dest_local", rare_dest_local)) return(false); - if (!getRareDestBitmapRedisField(key, "rare_dest_local_bg", rare_dest_local_bg)) return(false); + if (!getRareDestBitmapRedisField(key, "rare_dest_local_bg", rare_dest_local_bg)) { + ndpi_bitmap_free(rare_dest_local); + return(false); + } - if (!getRareDestBitmapRedisField(key, "rare_dest_remote", rare_dest_remote)) return(false); - if (!getRareDestBitmapRedisField(key, "rare_dest_remote_bg", rare_dest_remote_bg)) return(false); + if (!getRareDestBitmapRedisField(key, "rare_dest_remote", rare_dest_remote)) { + ndpi_bitmap_free(rare_dest_local); + ndpi_bitmap_free(rare_dest_local_bg); + return(false); + } + if (!getRareDestBitmapRedisField(key, "rare_dest_remote_bg", rare_dest_remote_bg)) { + ndpi_bitmap_free(rare_dest_local); + ndpi_bitmap_free(rare_dest_local_bg); + ndpi_bitmap_free(rare_dest_remote); + return(false); + } return(true); } From e3a555a94dd3fa0d62fe25885cfb8ededfb5ff1f Mon Sep 17 00:00:00 2001 From: Yuriy Rymarchuk Date: Tue, 19 Sep 2023 14:34:51 +0200 Subject: [PATCH 50/50] Fix: fixed memory leaks for bitmap initialization --- include/NetworkInterface.h | 12 ++--- src/NetworkInterface.cpp | 75 ++++++++++++++++------------- src/flow_checks/RareDestination.cpp | 3 +- 3 files changed, 49 insertions(+), 41 deletions(-) diff --git a/include/NetworkInterface.h b/include/NetworkInterface.h index 17e174c85ef8..4fe1d0a69d0e 100644 --- a/include/NetworkInterface.h +++ b/include/NetworkInterface.h @@ -452,8 +452,8 @@ class NetworkInterface : public NetworkInterfaceAlertableEntity { void setRareDestStructRedisField(const char *key, const char *field, u_int64_t value); bool getRareDestStructRedisField(const char *key, const char *field, u_int64_t *value); - void setRareDestBitmapRedisField(const char *key, const char *field, ndpi_bitmap *bitmap); - bool getRareDestBitmapRedisField(const char *key, const char *field, ndpi_bitmap *bitmap); + void setRareDestBitmapRedisField(const char *key, const char *field, ndpi_bitmap **bitmap); + bool getRareDestBitmapRedisField(const char *key, const char *field, ndpi_bitmap **bitmap); public: /** @@ -1394,13 +1394,13 @@ class NetworkInterface : public NetworkInterfaceAlertableEntity { /*RareDestination method implementation*/ - inline void setLocalRareDestBitmap(u_int32_t hash) { ndpi_bitmap_set(rare_dest_local, hash); } - inline void setRemoteRareDestBitmap(u_int32_t hash) { ndpi_bitmap_set(rare_dest_remote, hash); } + inline void setLocalRareDestBitmap(u_int32_t hash) { if(rare_dest_local) ndpi_bitmap_set(rare_dest_local, hash); } + inline void setRemoteRareDestBitmap(u_int32_t hash) { if(rare_dest_remote) ndpi_bitmap_set(rare_dest_remote, hash); } inline bool isSetLocalRareDestBitmap(u_int32_t hash) const { return ndpi_bitmap_isset(rare_dest_local, hash); } inline bool isSetRemoteRareDestBitmap(u_int32_t hash) const { return ndpi_bitmap_isset(rare_dest_remote, hash); } - inline void setLocalRareDestBitmap_BG(u_int32_t hash) { ndpi_bitmap_set(rare_dest_local_bg, hash); } - inline void setRemoteRareDestBitmap_BG(u_int32_t hash) { ndpi_bitmap_set(rare_dest_remote_bg, hash); } + inline void setLocalRareDestBitmap_BG(u_int32_t hash) { if(rare_dest_local_bg) ndpi_bitmap_set(rare_dest_local_bg, hash); } + inline void setRemoteRareDestBitmap_BG(u_int32_t hash) { if(rare_dest_remote_bg) ndpi_bitmap_set(rare_dest_remote_bg, hash); } inline bool getRareDestInitialTraining() const { return rareDestTraining.initialTraining;} inline void endRareDestInitialTraining(time_t t) { rareDestTraining.initialTraining = false; rareDestTraining.checkPoint = t; } diff --git a/src/NetworkInterface.cpp b/src/NetworkInterface.cpp index 3c4a05f401ae..ea7ea9cdb00d 100644 --- a/src/NetworkInterface.cpp +++ b/src/NetworkInterface.cpp @@ -221,6 +221,15 @@ NetworkInterface::NetworkInterface(const char *name, } #endif + if( id >= 0 && (!loadRareDestFromRedis() || (( time(NULL) - rareDestTraining.checkPoint ) > (2 * RARE_DEST_BACKGROUND_TRAINING_DURATION))) ){ + rareDestTraining.checkPoint = 0; + rareDestTraining.initialTraining = true; + rare_dest_local = ndpi_bitmap_alloc(); + rare_dest_local_bg = ndpi_bitmap_alloc(); + rare_dest_remote = ndpi_bitmap_alloc(); + rare_dest_remote_bg = ndpi_bitmap_alloc(); + } + is_loopback = (strncmp(ifname, "lo", 2) == 0) ? true : false; updateTrafficMirrored(); @@ -375,16 +384,6 @@ void NetworkInterface::init(const char *interface_name) { customFlowLuaScript_end = NULL; customHostLuaScript = NULL; - if( !loadRareDestFromRedis() || (( time(NULL) - rareDestTraining.checkPoint ) > (2 * RARE_DEST_BACKGROUND_TRAINING_DURATION)) ){ - rareDestTraining.checkPoint = 0; - rareDestTraining.initialTraining = true; - - rare_dest_local = ndpi_bitmap_alloc(); - rare_dest_local_bg = ndpi_bitmap_alloc(); - rare_dest_remote = ndpi_bitmap_alloc(); - rare_dest_remote_bg = ndpi_bitmap_alloc(); - } - INTERFACE_PROFILING_INIT(); } @@ -1033,11 +1032,13 @@ NetworkInterface::~NetworkInterface() { if (smart_recording_instance_name) free(smart_recording_instance_name); - saveRareDestToRedis(); - if(rare_dest_local) ndpi_bitmap_free(rare_dest_local); - if(rare_dest_local_bg) ndpi_bitmap_free(rare_dest_local_bg); - if(rare_dest_remote) ndpi_bitmap_free(rare_dest_remote); - if(rare_dest_remote_bg) ndpi_bitmap_free(rare_dest_remote_bg); + if ( id >= 0) { + saveRareDestToRedis(); + if(rare_dest_local) ndpi_bitmap_free(rare_dest_local); + if(rare_dest_local_bg) ndpi_bitmap_free(rare_dest_local_bg); + if(rare_dest_remote) ndpi_bitmap_free(rare_dest_remote); + if(rare_dest_remote_bg) ndpi_bitmap_free(rare_dest_remote_bg); + } } /* **************************************************** */ @@ -12390,11 +12391,14 @@ void NetworkInterface::setRareDestStructRedisField(const char *key, const char * ntop->getRedis()->hashSet(key, field, buf); } -void NetworkInterface::setRareDestBitmapRedisField(const char *key, const char *field, ndpi_bitmap *bitmap){ - char *value; +void NetworkInterface::setRareDestBitmapRedisField(const char *key, const char *field, ndpi_bitmap **bitmap){ + char *value = NULL; size_t size; - size = ndpi_bitmap_serialize(bitmap, &value); + size = ndpi_bitmap_serialize((*bitmap), &value); + + if( value == NULL ) return; + char *encoded_bmap = value ? Utils::base64_encode((unsigned char *)value, size) : NULL; if (encoded_bmap != NULL) { @@ -12413,15 +12417,15 @@ void NetworkInterface::setRareDestBitmapRedisField(const char *key, const char * void NetworkInterface::saveRareDestToRedis() { char key[CONST_MAX_LEN_REDIS_KEY]; - if((!ntop->getRedis()) || (!rare_dest_local) || (!rare_dest_remote)) return; + if((!ntop->getRedis()) || (!rare_dest_local) || (!rare_dest_remote) || (!rare_dest_local_bg) || (!rare_dest_remote_bg)) return; snprintf(key, sizeof(key), IFACE_RARE_DEST_SERIALIZED_KEY, get_id()); - setRareDestBitmapRedisField(key, "rare_dest_local", rare_dest_local); - setRareDestBitmapRedisField(key, "rare_dest_local_bg", rare_dest_local_bg); + setRareDestBitmapRedisField(key, "rare_dest_local", &rare_dest_local); + setRareDestBitmapRedisField(key, "rare_dest_local_bg", &rare_dest_local_bg); - setRareDestBitmapRedisField(key, "rare_dest_remote", rare_dest_remote); - setRareDestBitmapRedisField(key, "rare_dest_remote_bg", rare_dest_remote_bg); + setRareDestBitmapRedisField(key, "rare_dest_remote", &rare_dest_remote); + setRareDestBitmapRedisField(key, "rare_dest_remote_bg", &rare_dest_remote_bg); setRareDestStructRedisField(key, "checkPoint", (u_int64_t)rareDestTraining.checkPoint); setRareDestStructRedisField(key, "initialTraining", (u_int64_t)(rareDestTraining.initialTraining ? 1 : 0)); @@ -12434,8 +12438,8 @@ bool NetworkInterface::getRareDestStructRedisField(const char *key, const char * return(true); } -bool NetworkInterface::getRareDestBitmapRedisField(const char *key, const char *field, ndpi_bitmap *bitmap){ - char *bitmap_field_val; +bool NetworkInterface::getRareDestBitmapRedisField(const char *key, const char *field, ndpi_bitmap **bitmap){ + char *bitmap_field_val = NULL; u_int64_t value; size_t size; @@ -12444,26 +12448,29 @@ bool NetworkInterface::getRareDestBitmapRedisField(const char *key, const char * if (!getRareDestStructRedisField(key, len_field, &value)) return(false); size = (size_t)(value); - + if((bitmap_field_val = (char *) malloc(size)) == NULL) { ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to allocate memory to deserialize %s", key); return(false); } - if(ntop->getRedis()->hashGet(key, field, bitmap_field_val, size) != 0) { + if(ntop->getRedis()->hashGet(key, field, bitmap_field_val, (size)) != 0) { free(bitmap_field_val); return(false); } - bitmap = ndpi_bitmap_deserialize((char *)Utils::base64_decode((std::string)bitmap_field_val).c_str()); - free(bitmap_field_val); + if( bitmap_field_val == NULL ) return(false); + (*bitmap) = ndpi_bitmap_deserialize((char *)(Utils::base64_decode((std::string)bitmap_field_val).c_str())); + + free(bitmap_field_val); return(true); } bool NetworkInterface::loadRareDestFromRedis() { char key[CONST_MAX_LEN_REDIS_KEY]; u_int64_t value; + rare_dest_local = rare_dest_local_bg = rare_dest_remote = rare_dest_remote_bg = NULL; if((!ntop->getRedis())) return(false); @@ -12475,18 +12482,20 @@ bool NetworkInterface::loadRareDestFromRedis() { if (!getRareDestStructRedisField(key, "initialTraining", &value)) return(false); rareDestTraining.initialTraining = value ? true : false; - if (!getRareDestBitmapRedisField(key, "rare_dest_local", rare_dest_local)) return(false); - if (!getRareDestBitmapRedisField(key, "rare_dest_local_bg", rare_dest_local_bg)) { + if (!getRareDestBitmapRedisField(key, "rare_dest_local", &rare_dest_local)) return(false); + + if (!getRareDestBitmapRedisField(key, "rare_dest_local_bg", &rare_dest_local_bg)) { ndpi_bitmap_free(rare_dest_local); return(false); } - if (!getRareDestBitmapRedisField(key, "rare_dest_remote", rare_dest_remote)) { + if (!getRareDestBitmapRedisField(key, "rare_dest_remote", &rare_dest_remote)) { ndpi_bitmap_free(rare_dest_local); ndpi_bitmap_free(rare_dest_local_bg); return(false); } - if (!getRareDestBitmapRedisField(key, "rare_dest_remote_bg", rare_dest_remote_bg)) { + + if (!getRareDestBitmapRedisField(key, "rare_dest_remote_bg", &rare_dest_remote_bg)) { ndpi_bitmap_free(rare_dest_local); ndpi_bitmap_free(rare_dest_local_bg); ndpi_bitmap_free(rare_dest_remote); diff --git a/src/flow_checks/RareDestination.cpp b/src/flow_checks/RareDestination.cpp index 708a68a8b906..0cdbd76a5313 100644 --- a/src/flow_checks/RareDestination.cpp +++ b/src/flow_checks/RareDestination.cpp @@ -56,7 +56,7 @@ void RareDestination::protocolDetected(Flow *f) { bool is_rare_destination = false; - if(f->get_cli_host()->isLocalHost() && (f->getFlowServerInfo() != NULL || !f->get_srv_ip_addr()->isEmpty()) ) { + if( f->get_cli_host()->isLocalHost() && (f->getFlowServerInfo() != NULL || !f->get_srv_ip_addr()->isEmpty()) ) { time_t t_now = time(NULL); NetworkInterface *iface = f->getInterface(); @@ -67,7 +67,6 @@ void RareDestination::protocolDetected(Flow *f) { /* initial training */ if (iface->getRareDestInitialTraining()) { - destType == 0 ? iface->setLocalRareDestBitmap(hash) : iface->setRemoteRareDestBitmap(hash); if (!iface->getRareDestTrainingCheckPoint())