Rare/Unusual Domain & Destination Flow Check

include/NetworkInterface.h

@@ -106,6 +106,19 @@ class NetworkInterface : public NetworkInterfaceAlertableEntity {
     u_int32_t local_hosts, remote_hosts;
   } tot_num_anomalies;
   AlertsQueue *alertsQueue;
+
+  /* RareDestination data implementation*/
+  ndpi_bitmap *rare_dest_local;
+  ndpi_bitmap *rare_dest_remote;
+  ndpi_bitmap *rare_dest_local_bg;
+  ndpi_bitmap *rare_dest_remote_bg;
+
+  struct {
+    bool initialTraining;
+    time_t checkPoint;
+  } rareDestTraining;
+  /***************************************/
+
 #if defined(NTOPNG_PRO)
   PeriodicityMap *pMap;
   ServiceMap *sMap;
@@ -433,6 +446,15 @@ class NetworkInterface : public NetworkInterfaceAlertableEntity {
   void build_protocol_flow_stats_lua_rsp(lua_State *vm, AggregatedFlowsStats *fs,
                                          u_int32_t size, u_int *num);
 
+  void saveRareDestToRedis();
+  bool loadRareDestFromRedis();
+
+  void setRareDestStructRedisField(const char *key, const char *field, u_int64_t value);
+  bool getRareDestStructRedisField(const char *key, const char *field, u_int64_t *value);
+
+  void setRareDestBitmapRedisField(const char *key, const char *field, ndpi_bitmap **bitmap);
+  bool getRareDestBitmapRedisField(const char *key, const char *field, ndpi_bitmap **bitmap);
+
 public:
   /**
    * @brief A Constructor
@@ -1370,6 +1392,34 @@ class NetworkInterface : public NetworkInterfaceAlertableEntity {
 				   void *user_data,
 				   bool *matched);
 
+  /*RareDestination method implementation*/
+
+  inline void setLocalRareDestBitmap(u_int32_t hash)            { if(rare_dest_local) ndpi_bitmap_set(rare_dest_local, hash); }
+  inline void setRemoteRareDestBitmap(u_int32_t hash)           { if(rare_dest_remote) ndpi_bitmap_set(rare_dest_remote, hash); }
+  inline bool isSetLocalRareDestBitmap(u_int32_t hash) const    { return ndpi_bitmap_isset(rare_dest_local, hash); }
+  inline bool isSetRemoteRareDestBitmap(u_int32_t hash) const   { return ndpi_bitmap_isset(rare_dest_remote, hash); }
+
+  inline void setLocalRareDestBitmap_BG(u_int32_t hash)         { if(rare_dest_local_bg) ndpi_bitmap_set(rare_dest_local_bg, hash); }
+  inline void setRemoteRareDestBitmap_BG(u_int32_t hash)        { if(rare_dest_remote_bg) ndpi_bitmap_set(rare_dest_remote_bg, hash); }
+
+  inline bool getRareDestInitialTraining() const                { return rareDestTraining.initialTraining;}
+  inline void endRareDestInitialTraining(time_t t)              { rareDestTraining.initialTraining = false; rareDestTraining.checkPoint = t; }
+
+  inline time_t getRareDestTrainingCheckPoint() const           { return rareDestTraining.checkPoint; }
+  inline void setRareDestTrainingCheckPoint(time_t t)           { rareDestTraining.checkPoint = t; }
+
+  void swapRareDestBitmaps() {
+    ndpi_bitmap_free(rare_dest_local);
+    rare_dest_local = rare_dest_local_bg;
+    rare_dest_local_bg = ndpi_bitmap_alloc();
+
+    ndpi_bitmap_free(rare_dest_remote);
+    rare_dest_remote = rare_dest_remote_bg;
+    rare_dest_remote_bg = ndpi_bitmap_alloc();
+  }
+
+  /***************************************/
+
 #ifdef NTOPNG_PRO
   static bool compute_client_server_flow_stats(GenericHashEntry *node,
                                                void *user_data, bool *matched);

include/flow_checks/RareDestination.h

@@ -25,6 +25,9 @@
 #include "ntop_includes.h"
 
 class RareDestination : public FlowCheck {
+ private:
+  u_int32_t getDestinationHash(Flow *f, u_int8_t *destType);
+
  public:
   RareDestination()
       : FlowCheck(ntopng_edition_community, true /* Packet Interfaces only */,

include/ntop_defines.h

@@ -1505,4 +1505,10 @@ extern struct ntopngLuaContext *getUserdata(struct lua_State *vm);
 #define OFFLINE_LOCAL_HOSTS_KEY "ntopng.hosts.offline.ifid_%d"
 /******************************************************************************/
 
+#define IFACE_RARE_DEST_SERIALIZED_KEY "ntopng.iface_rare_dest_fields.ifid_%d"
+#define RARE_DEST_INITIAL_TRAINING_DURATION 86400  /* seconds ( 1 day  ) */
+#define RARE_DEST_BACKGROUND_TRAINING_DURATION 604800  /* seconds ( 1 week  ) */
+
+/******************************************************************************/
+
 #endif /* _NTOP_DEFINES_H_ */

src/NetworkInterface.cpp

@@ -221,6 +221,15 @@ NetworkInterface::NetworkInterface(const char *name,
   }
 #endif
 
+  if( id >= 0 && (!loadRareDestFromRedis() || (( time(NULL) - rareDestTraining.checkPoint ) > (2 * RARE_DEST_BACKGROUND_TRAINING_DURATION))) ){
+    rareDestTraining.checkPoint = 0;
+    rareDestTraining.initialTraining = true;
+    rare_dest_local = ndpi_bitmap_alloc();
+    rare_dest_local_bg = ndpi_bitmap_alloc();
+    rare_dest_remote = ndpi_bitmap_alloc();
+    rare_dest_remote_bg = ndpi_bitmap_alloc();
+  }
+
   is_loopback = (strncmp(ifname, "lo", 2) == 0) ? true : false;
 
   updateTrafficMirrored();
@@ -1022,6 +1031,14 @@ NetworkInterface::~NetworkInterface() {
   cleanShadownDPI();
 
   if (smart_recording_instance_name) free(smart_recording_instance_name);
+
+  if ( id >= 0) {
+    saveRareDestToRedis();
+    if(rare_dest_local) ndpi_bitmap_free(rare_dest_local);
+    if(rare_dest_local_bg) ndpi_bitmap_free(rare_dest_local_bg);
+    if(rare_dest_remote) ndpi_bitmap_free(rare_dest_remote);
+    if(rare_dest_remote_bg) ndpi_bitmap_free(rare_dest_remote_bg);
+  }
 }
 
 /* **************************************************** */
@@ -12365,3 +12382,125 @@ void NetworkInterface::getActiveMacs(lua_State *vm) {
   walker(&begin_slot, true /* walk_all */, walker_macs,
          active_mac_search_walker, (void *)&s);
 }
+
+/* *************************************** */
+
+void NetworkInterface::setRareDestStructRedisField(const char *key, const char *field, u_int64_t value){
+  char buf[32];
+  snprintf(buf, sizeof(buf), "%lu", value);
+  ntop->getRedis()->hashSet(key, field, buf);
+}
+
+void NetworkInterface::setRareDestBitmapRedisField(const char *key, const char *field, ndpi_bitmap **bitmap){
+  char *value = NULL;
+  size_t size;
+
+  size = ndpi_bitmap_serialize((*bitmap), &value);
+
+  if( value == NULL ) return;
+
+  char *encoded_bmap = value ? Utils::base64_encode((unsigned char *)value, size) : NULL;
+
+  if (encoded_bmap != NULL) {
+    size = strlen(encoded_bmap);
+    ntop->getRedis()->hashSet(key, field, encoded_bmap);
+
+    char len_field[32];
+    snprintf(len_field, sizeof(len_field), "%s_len", field);
+    setRareDestStructRedisField(key, len_field, (u_int64_t)size);
+
+    free(encoded_bmap);
+    free(value);
+  }
+}
+
+void NetworkInterface::saveRareDestToRedis() {
+  char key[CONST_MAX_LEN_REDIS_KEY];
+
+  if((!ntop->getRedis()) || (!rare_dest_local) || (!rare_dest_remote) || (!rare_dest_local_bg) || (!rare_dest_remote_bg)) return;
+
+  snprintf(key, sizeof(key), IFACE_RARE_DEST_SERIALIZED_KEY, get_id());
+
+  setRareDestBitmapRedisField(key, "rare_dest_local", &rare_dest_local);
+  setRareDestBitmapRedisField(key, "rare_dest_local_bg", &rare_dest_local_bg);
+
+  setRareDestBitmapRedisField(key, "rare_dest_remote", &rare_dest_remote);
+  setRareDestBitmapRedisField(key, "rare_dest_remote_bg", &rare_dest_remote_bg);
+
+  setRareDestStructRedisField(key, "checkPoint", (u_int64_t)rareDestTraining.checkPoint);
+  setRareDestStructRedisField(key, "initialTraining", (u_int64_t)(rareDestTraining.initialTraining ? 1 : 0));
+}
+
+bool NetworkInterface::getRareDestStructRedisField(const char *key, const char *field, u_int64_t *value){
+  char buf[32];
+  if( ntop->getRedis()->hashGet(key, field, buf, sizeof(buf)) != 0 ) return(false);
+  *value = strtoul(buf, NULL, 10);
+  return(true);
+}
+
+bool NetworkInterface::getRareDestBitmapRedisField(const char *key, const char *field, ndpi_bitmap **bitmap){
+  char *bitmap_field_val = NULL;
+  u_int64_t value;
+  size_t size;
+
+  char len_field[32];
+  snprintf(len_field, sizeof(len_field), "%s_len", field);
+
+  if (!getRareDestStructRedisField(key, len_field, &value)) return(false);
+  size = (size_t)(value);
+
+  if((bitmap_field_val = (char *) malloc(size)) == NULL) {
+    ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to allocate memory to deserialize %s", key);
+    return(false);
+  }
+
+  if(ntop->getRedis()->hashGet(key, field, bitmap_field_val, (size)) != 0) {
+    free(bitmap_field_val);
+    return(false);
+  }
+
+  if( bitmap_field_val == NULL ) return(false);
+
+  (*bitmap) = ndpi_bitmap_deserialize((char *)(Utils::base64_decode((std::string)bitmap_field_val).c_str()));
+
+  free(bitmap_field_val);
+  return(true);
+}
+
+bool NetworkInterface::loadRareDestFromRedis() {
+  char key[CONST_MAX_LEN_REDIS_KEY];
+  u_int64_t value;
+  rare_dest_local = rare_dest_local_bg = rare_dest_remote = rare_dest_remote_bg = NULL;
+
+  if((!ntop->getRedis())) return(false); 
+
+  snprintf(key, sizeof(key), IFACE_RARE_DEST_SERIALIZED_KEY, get_id());
+
+  if (!getRareDestStructRedisField(key, "checkPoint", &value)) return(false);
+  rareDestTraining.checkPoint = (time_t)(value);
+
+  if (!getRareDestStructRedisField(key, "initialTraining", &value)) return(false);
+  rareDestTraining.initialTraining = value ? true : false;
+
+  if (!getRareDestBitmapRedisField(key, "rare_dest_local", &rare_dest_local)) return(false);
+
+  if (!getRareDestBitmapRedisField(key, "rare_dest_local_bg", &rare_dest_local_bg)) {
+    ndpi_bitmap_free(rare_dest_local);
+    return(false);
+  }
+
+  if (!getRareDestBitmapRedisField(key, "rare_dest_remote", &rare_dest_remote)) {
+    ndpi_bitmap_free(rare_dest_local);
+    ndpi_bitmap_free(rare_dest_local_bg);
+    return(false);
+  }
+
+  if (!getRareDestBitmapRedisField(key, "rare_dest_remote_bg", &rare_dest_remote_bg)) {
+    ndpi_bitmap_free(rare_dest_local);
+    ndpi_bitmap_free(rare_dest_local_bg);
+    ndpi_bitmap_free(rare_dest_remote);
+    return(false);
+  }
+
+  return(true);
+}

src/flow_checks/RareDestination.cpp

@@ -24,16 +24,79 @@
 
 /* ***************************************************** */
 
+u_int32_t RareDestination::getDestinationHash(Flow *f, u_int8_t *destType) {
+
+  u_int32_t hash = 0;
+
+  if (f->isLocalToLocal()) {
+
+    *destType = 0;  // local destination
+    LocalHost *ldest = (LocalHost*)f->get_srv_host();
+
+    if (!ldest->isLocalUnicastHost() && ldest->isDHCPHost()) {
+      Mac *mac = ldest->getMac();
+      hash = mac ? mac->key() : 0;
+    }
+    else if (ldest->isIPv6() || ldest->isIPv4()) {
+      hash =  ldest->get_ip()->key();
+    }
+
+  }
+  else if (f->isLocalToRemote()) {
+    *destType = 1;  // remote destination
+    hash = Utils::hashString(f->getFlowServerInfo());
+  }
+
+  return hash;
+}
+
+/* ***************************************************** */
+
 void RareDestination::protocolDetected(Flow *f) {
+
   bool is_rare_destination = false;
 
-  /* TODO: check if this is a real rare destination */
-  if (f->getFlowServerInfo() != NULL) {
-#ifdef TODO_HERE
-    ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** Rare destination %s",
-                                 f->getFlowServerInfo());
-    is_rare_destination = true;
-#endif
+  if( f->get_cli_host()->isLocalHost() && (f->getFlowServerInfo() != NULL || !f->get_srv_ip_addr()->isEmpty()) ) {
+
+    time_t t_now = time(NULL);
+    NetworkInterface *iface = f->getInterface();
+
+    u_int8_t destType;
+    u_int32_t hash = getDestinationHash(f, &destType);
+    if(hash == 0) return;
+
+    /* initial training */
+    if (iface->getRareDestInitialTraining()) {
+      destType == 0 ? iface->setLocalRareDestBitmap(hash) : iface->setRemoteRareDestBitmap(hash);
+
+      if (!iface->getRareDestTrainingCheckPoint())
+        iface->setRareDestTrainingCheckPoint(t_now);
+
+      else if (t_now - iface->getRareDestTrainingCheckPoint() >= RARE_DEST_INITIAL_TRAINING_DURATION)
+        iface->endRareDestInitialTraining(t_now);
+
+      return;
+    }
+
+    /* background training */
+    destType == 0 ? iface->setLocalRareDestBitmap_BG(hash) : iface->setRemoteRareDestBitmap_BG(hash);
+
+    /* update bitmap */
+    if (destType == 0 && !iface->isSetLocalRareDestBitmap(hash)) {
+      is_rare_destination = true;
+      iface->setLocalRareDestBitmap(hash);
+    }
+
+    else if (destType == 1 && !iface->isSetRemoteRareDestBitmap(hash)) {
+      is_rare_destination = true;
+      iface->setRemoteRareDestBitmap(hash);
+    }
+
+    /* check if background training has to restart */
+    if (t_now - iface->getRareDestTrainingCheckPoint() >= RARE_DEST_BACKGROUND_TRAINING_DURATION) {
+      iface->setRareDestTrainingCheckPoint(t_now);
+      iface->swapRareDestBitmaps();
+    }
   }
 
   if (is_rare_destination) {