Merge pull request #9 from ntop/dev

sync

configure.seed

@@ -269,7 +269,9 @@ DUMMY=`cd ./pro; make build`
 PRO_INCS="${PRO_INCS} -I${PWD}/pro -I${PWD}/pro/utils -I${PWD}/pro/third-party/libb64-1.2.1/include"
 
 if test -r "../license/systemId.c"; then :
+ LICENSELIBS="-L`pwd`/../license -llicense"
  AC_DEFINE_UNQUOTED(NTOPNG_PRO_HAVE_LICENSE, 1, [ntopng has license])
+ PRO_LIBS="${PRO_LIBS} ${LICENSELIBS}" 
 fi
 else
 AC_MSG_RESULT(not found)

include/AddressTree.h

@@ -57,7 +57,7 @@ class AddressTree {
   bool addAddresses(char *net, const int16_t user_data = -1);
   void getAddresses(lua_State* vm) const;
   int16_t findAddress(int family, void *addr, u_int8_t *network_mask_bits = NULL);
-  int16_t findMac(u_int8_t addr[]);
+  int16_t findMac(const u_int8_t addr[]);
   bool match(char *addr);
   bool match(const IpAddress * const ipa, int network_bits) const;
   void dump();

include/FrequentTrafficItems.h

@@ -74,7 +74,7 @@ class FrequentTrafficItems {
   void addPoolProtocol(u_int16_t pool_id, u_int16_t proto_id, u_int32_t value);
   void luaTopPoolsProtocols(lua_State *vm);
 
-  void addMacProtocol(u_int8_t mac[6], u_int16_t proto_id, u_int32_t value);
+  void addMacProtocol(const u_int8_t mac[6], u_int16_t proto_id, u_int32_t value);
   void luaTopMacsProtocols(lua_State *vm);
 };
 

include/Host.h

@@ -152,7 +152,7 @@ class Host : public GenericHashEntry {
   void set_mac(u_int8_t *m);
   inline bool isBlacklisted()                  { return(blacklisted_host);  };
   void reloadHostBlacklist();
-  inline u_int8_t*  get_mac()                  { return(mac ? mac->get_mac() : NULL);      }
+  inline const u_int8_t* const get_mac() const { return(mac ? mac->get_mac() : NULL);}
   inline Mac* getMac() const                   { return(mac);              }
   char * getResolvedName(char * const buf, ssize_t buf_len);
   char * getMDNSName(char * const buf, ssize_t buf_len);

include/HostPools.h

@@ -85,7 +85,7 @@ class HostPools {
   u_int16_t getPool(Mac *m);
 
   bool findIpPool(IpAddress *ip, u_int16_t vlan_id, u_int16_t *found_pool, patricia_node_t **found_node);
-  bool findMacPool(u_int8_t *mac, u_int16_t *found_pool);
+  bool findMacPool(const u_int8_t * const mac, u_int16_t *found_pool);
   bool findMacPool(Mac *mac, u_int16_t *found_pool);
   void lua(lua_State *vm);
 

include/Mac.h

@@ -85,7 +85,7 @@ class Mac : public GenericHashEntry {
 
   MacLocation locate();
   inline u_int32_t key()                       { return(Utils::macHash(mac)); }
-  inline u_int8_t* get_mac()                   { return(mac);                 }
+  inline const u_int8_t* const get_mac() const { return(mac);                 }
   inline const char * const get_manufacturer() { return manuf ? manuf : NULL; }
   bool isNull() const;
 

include/NetworkInterface.h

@@ -255,6 +255,7 @@ class NetworkInterface : public Checkpointable {
 
   void topItemsCommit(const struct timeval *when);
   void checkMacIPAssociation(bool triggerEvent, u_char *_mac, u_int32_t ipv4);
+  bool checkBroadcastDomainTooLarge(u_int32_t bcast_mask, u_int16_t vlan_id, const Mac * const src_mac, const Mac * const dst_mac, u_int32_t spa, u_int32_t tpa) const;
   void pollQueuedeBPFEvents();
   void reloadCustomCategories();
 
@@ -522,7 +523,7 @@ class NetworkInterface : public Checkpointable {
 
   void runHousekeepingTasks();
   void runShutdownTasks();
-  ArpStatsMatrixElement* getArpHashMatrixElement(u_int8_t _src_mac[6], u_int8_t _dst_mac[6], bool * const src2dst);
+  ArpStatsMatrixElement* getArpHashMatrixElement(const u_int8_t _src_mac[6], const u_int8_t _dst_mac[6], bool * const src2dst);
   Vlan* getVlan(u_int16_t vlanId, bool createIfNotPresent);
   AutonomousSystem *getAS(IpAddress *ipa, bool createIfNotPresent);
   Country* getCountry(const char *country_name, bool createIfNotPresent);

include/Ntop.h

@@ -77,7 +77,6 @@ class Ntop {
 #ifndef WIN32
   NagiosManager *nagios_manager;
 #endif
-  FlowChecker *flow_checker;
 #endif
 
   void loadLocalInterfaceAddress();
@@ -117,7 +116,6 @@ class Ntop {
   inline void rotateLogs(bool mode)                { getTrace()->rotate_logs(mode);     };
 #ifdef NTOPNG_PRO
   void registerNagios(void);
-  inline FlowChecker *getFlowChecker() { return(flow_checker); };
 #endif
 
   /**

include/Utils.h

@@ -107,7 +107,7 @@ class Utils {
   static const char* flowStatus2str(FlowStatus s, AlertType *aType, AlertLevel *aLevel);
   static char* formatMac(const u_int8_t * const mac, char *buf, u_int buf_len);
   static void  parseMac(u_int8_t *mac, const char *symMac);
-  static u_int32_t macHash(u_int8_t *mac);
+  static u_int32_t macHash(const u_int8_t * const mac);
   static bool isSpecialMac(u_int8_t *mac);
   static int numberOfSetBits(u_int32_t i);
   static void initRedis(Redis **r, const char *redis_host, const char *redis_password,

include/VlanAddressTree.h

@@ -36,7 +36,7 @@ class VlanAddressTree {
   bool addAddresses(u_int16_t vlan_id, char *net, const int16_t user_data = -1);
 
   int16_t findAddress(u_int16_t vlan_id, int family, void *addr, u_int8_t *network_mask_bits = NULL);
-  int16_t findMac(u_int16_t vlan_id, u_int8_t addr[]);
+  int16_t findMac(u_int16_t vlan_id, const u_int8_t addr[]);
 
   inline AddressTree *getAddressTree(u_int16_t vlan_id) { return tree[vlan_id]; };
 };

include/ntop_defines.h

@@ -498,10 +498,11 @@
 #define CONST_INFLUXDB_FILE_QUEUE          "ntopng.influx_file_queue"
 #define CONST_INFLUXDB_FLUSH_TIME          10 /* sec */
 #define CONST_INFLUXDB_MAX_DUMP_SIZE       4194304 /* 4 MB */
-#define CONST_ALERT_MSG_QUEUE              "ntopng.alert_queue"
-#define CONST_ALERT_MAC_IP_QUEUE           "ntopng.alert_mac_ip_queue"
-#define CONST_ALERT_NFQ_FLUSHED            "ntopng.alert_nfq_flushed_queue"
-#define CONST_ALERT_HOST_REMOTE_TO_REMOTE  "ntopng.alert_host_remote_to_remote"
+#define CONST_ALERT_MSG_QUEUE                    "ntopng.alert_queue"
+#define CONST_ALERT_MAC_IP_QUEUE                 "ntopng.alert_mac_ip_queue"
+#define CONST_ALERT_NFQ_FLUSHED                  "ntopng.alert_nfq_flushed_queue"
+#define CONST_ALERT_HOST_REMOTE_TO_REMOTE        "ntopng.alert_host_remote_to_remote"
+#define CONST_ALERT_BCAST_DOMAIN_TOO_LARGE_QUEUE "ntopng.alert_bcast_domain_too_large"
 #define CONST_REMOTE_TO_REMOTE_MAX_QUEUE   32
 #define CONST_SQL_QUEUE                    "ntopng.sql_queue"
 #define CONST_SQL_BATCH_SIZE               32

include/ntop_includes.h

@@ -299,7 +299,6 @@ using namespace std;
 #ifndef WIN32
 #include "NagiosManager.h"
 #endif
-#include "FlowChecker.h"
 #include "FrequentStringItems.h"
 #include "FrequentNumericItems.h"
 #include "FrequentTrafficItems.h"

packages/ntopng.spec.in

@@ -44,7 +44,7 @@ mkdir -p $RPM_BUILD_ROOT/etc/sudoers.d/
 mkdir -p $RPM_BUILD_ROOT/etc/init.d 
 %endif
 cp $HOME/ntopng/ntopng $RPM_BUILD_ROOT/usr/bin
-strip $RPM_BUILD_ROOT/usr/bin/ntopng
+# strip $RPM_BUILD_ROOT/usr/bin/ntopng
 cp $HOME/ntopng/ntopng.8 $RPM_BUILD_ROOT/usr/share/man/man8/ 
 cp -Lr $HOME/ntopng/httpdocs $HOME/ntopng/scripts $RPM_BUILD_ROOT/usr/share/ntopng  # L to dereference symlinks
 mv $RPM_BUILD_ROOT/usr/share/ntopng/httpdocs/misc/ntopng-utils-manage-config $RPM_BUILD_ROOT/usr/bin

packages/ubuntu/debian.ntopng/rules.in

@@ -43,7 +43,7 @@ binary-arch: build install
 	dh_installdebconf
 	dh_installman
 # Do not strip binary so we can leave debug symbols
-	dh_strip
+#	dh_strip
 	dh_compress
 #	dh_fixperms
 	dh_installdeb
@@ -60,7 +60,7 @@ binary-arch: build install
 	fi
 	rm -rf ./debian/@APP@/etc/ntopng/nedge.conf
 	rm -rf ./debian/@APP@/usr/share/ntopng/httpdocs/geoip/*.dat
-	strip ./debian/@APP@/usr/bin/@APP@
+#	strip ./debian/@APP@/usr/bin/@APP@
 	-find ./debian/@APP@ -name .svn -exec /bin/rm -rf {} ';'
 	-find ./debian/@APP@ -name '*~' -exec /bin/rm -rf {} ';'
 	-find ./debian/@APP@ -name '*#' -exec /bin/rm -rf {} ';'

scripts/callbacks/system/housekeeping.lua

@@ -22,6 +22,7 @@ if ntop.isnEdge() then
    check_nfq_flushed_queue_alerts()
 end
 check_host_remote_to_remote_alerts()
+check_broadcast_domain_too_large_alerts()
 check_process_alerts()
 callback_utils.uploadTSdata()
 lists_utils.checkReloadLists()

scripts/locales/en.lua

@@ -224,6 +224,7 @@ local lang = {
     ["influxdb_write_error"] = "There was an error while sending timeseries data to \"%{influxdb}\": %{err}",
     ["interface_entity"] = "interface %{entity_value}",
     ["mac_ip_association_change"] = "IP %{ip} changed association from <a href=\"%{old_mac_url}\">%{old_mac}</a> to <a href=\"%{new_mac_url}\">%{new_mac}</a>",
+    ["broadcast_domain_too_large"] = "ARP traffic from <a href=\"%{src_mac_url}\">%{src_mac}</a>/<a href=\"%{spa_url}\">%{spa}</a> to <a href=\"%{dst_mac_url}\">%{dst_mac}</a>/<a href=\"%{tpa_url}\">%{tpa}</a> detected. It is unlikely to see ARP traffic between those IPs as they are seemingly belonging to different broadcast domains. Check for hosts and networks configurations.",
     ["network_entity"] = "network %{entity_value}",
     ["nfq_flushed"] = "Interface <a href=\"%{url}\">%{name}</a> packets queue flushed. Queue %{pct}%% full with %{tot} packets and %{dropped} drops.",
     ["ntopng_anomalous_termination"] = "Started after anomalous termination (<a href=\"%{url}\">bug report</a>)",
@@ -259,6 +260,7 @@ local lang = {
     ["host_pool_connection"] = "Host Pool Connection",
     ["host_pool_disconnection"] = "Host Pool Disconnection",
     ["icmp_anomaly"] = "ICMP Anomaly",
+    ["broadcast_domain_too_large"] = "Broadcast domain",
     ["inactivity"] = "Inactivity",
     ["influxdb_export_failure"] = "InfluxDB Export Failure",
     ["info"] = "Info",

scripts/lua/modules/alert_consts.lua

@@ -54,6 +54,7 @@ alert_consts.alert_type_keys = {
    { "<i class='fa fa-sticky-note'></i> " .. i18n("alerts_dashboard.list_download_failed"),        31, "list_download_failed"       },
    { "<i class='fa fa-life-ring'></i> " .. i18n("alerts_dashboard.dns_anomaly"),                   32, "dns_anomaly"                },
    { "<i class='fa fa-life-ring'></i> " .. i18n("alerts_dashboard.icmp_anomaly"),                  33, "icmp_anomaly"               },
+   { "<i class='fa fa-sitemap'></i> " .. i18n("alerts_dashboard.broadcast_domain_too_large"),      34, "broadcast_domain_too_large" },
 }
 
 -- Keep in sync with ntop_typedefs.h:AlertEntity

scripts/lua/modules/alert_utils.lua

@@ -2670,6 +2670,10 @@ local function getMacUrl(mac)
    return ntop.getHttpPrefix() .. "/lua/mac_details.lua?host=" .. mac
 end
 
+local function getHostUrl(host, vlan_id)
+   return ntop.getHttpPrefix() .. "/lua/host_details.lua?" .. hostinfo2url({host = host, vlan = vlan_id})
+end
+
 local function getSavedDeviceNameKey(mac)
    return "ntopng.cache.devnames." .. mac
 end
@@ -2708,6 +2712,39 @@ function check_mac_ip_association_alerts()
    end   
 end
 
+-- Global function
+function check_broadcast_domain_too_large_alerts()
+   while(true) do
+      local message = ntop.lpopCache("ntopng.alert_bcast_domain_too_large")
+      local elems
+
+      if((message == nil) or (message == "")) then
+	 break
+      end
+
+      elems = json.decode(message)
+
+      if elems ~= nil then
+	 local entity = alertEntity("interface")
+	 local entity_value = "iface_"..elems.ifid
+
+	 --io.write(elems.ip.." ==> "..message.."[".. elems.ifname .."]\n")
+	 interface.select(elems.ifname)
+	 interface.storeAlert(entity, entity_value,
+			      alertType("broadcast_domain_too_large"),
+			      alertSeverity("warning"),
+			      i18n("alert_messages.broadcast_domain_too_large",
+				   {src_mac = elems.src_mac,
+				    src_mac_url = getMacUrl(elems.src_mac),
+				    dst_mac = elems.dst_mac,
+				    dst_mac_url = getMacUrl(elems.dst_mac),
+				    spa = elems.spa,
+				    spa_url = getHostUrl(elems.spa, elems.vlan_id),
+				    tpa = elems.tpa,
+				    tpa_url = getHostUrl(elems.tpa, elems.vlan_id)}))
+      end
+   end
+end
 
 -- Global function
 function check_nfq_flushed_queue_alerts()

scripts/lua/modules/http_lint.lua

@@ -182,6 +182,10 @@ local function validateUsername(p)
    return validateSingleWord(p)
 end
 
+local function licenseCleanup(p)
+   return p -- don't touch passwords (checks against valid fs paths already performed)
+end
+
 local function passwordCleanup(p)
    return p -- don't touch passwords (checks against valid fs paths already performed)
 end
@@ -197,6 +201,11 @@ local function whereCleanup(p)
    return(p:gsub('%W><!()','_'))
 end
 
+local function validateLicense(p)
+   -- A password (e.g. used in ntopng authentication)
+   return string.match(p,"[%l%u%d/+]+=*") == p or validateEmpty(p)
+end
+
 local function validatePassword(p)
    -- A password (e.g. used in ntopng authentication)
    return validateSingleWord(p)
@@ -1097,7 +1106,7 @@ local known_parameters = {
    ["row_id"]                  = validateNumber,                -- A number used to identify a record in a database
    ["rrd_file"]                = validateUnquoted,              -- A path or special identifier to read an RRD file
    ["port"]                    = validatePort,                  -- An application port
-   ["ntopng_license"]          = validateSingleWord,            -- ntopng licence string
+   ["ntopng_license"]          = {licenseCleanup, validateLicense},          -- ntopng licence string
    ["syn_attacker_threshold"]        = validateEmptyOr(validateNumber),
    ["global_syn_attacker_threshold"] = validateEmptyOr(validateNumber),
    ["syn_victim_threshold"]          = validateEmptyOr(validateNumber),

src/AddressTree.cpp

@@ -284,7 +284,7 @@ int16_t AddressTree::findAddress(int family, void *addr, u_int8_t *network_mask_
 
 /* ******************************************* */
 
-int16_t AddressTree::findMac(u_int8_t addr[]) {
+int16_t AddressTree::findMac(const u_int8_t addr[]) {
   MacKey_t *s = NULL;
 
   HASH_FIND(hh, macs, addr, 6, s);

src/Flow.cpp

@@ -545,11 +545,6 @@ void Flow::processDetectedProtocol() {
     break;
   } /* switch */
 
-#ifdef NTOPNG_PRO
-  if((ndpiDetectedProtocol.app_protocol == NDPI_PROTOCOL_UNKNOWN) && (!l7_protocol_guessed))
-    ntop->getFlowChecker()->flowCheck(this);
-#endif
-
   if(protocol_processed
      /* For DNS we delay the memory free so that we can let nDPI analyze all the packets of the flow */
      && (l7proto != NDPI_PROTOCOL_DNS))
@@ -561,8 +556,6 @@ void Flow::processDetectedProtocol() {
 void Flow::guessProtocol() {
   if(detection_completed)
     return; /* Nothing to do */
-  else
-    detection_completed = true; /* We give up */
 
   /* This code should no longer be necessary as the nDPI API changed */
   if((protocol == IPPROTO_TCP) || (protocol == IPPROTO_UDP)) {
@@ -586,6 +579,8 @@ void Flow::guessProtocol() {
 
     l7_protocol_guessed = true;
   }
+
+  detection_completed = true; /* We give up */
 }
 
 /* *************************************** */

src/FrequentTrafficItems.cpp

@@ -114,7 +114,7 @@ void FrequentTrafficItems::luaTopPoolsProtocols(lua_State *vm) {
 
 /* ******************************************************** */
 
-void FrequentTrafficItems::addMacProtocol(u_int8_t mac[6], u_int16_t proto_id, u_int32_t value) {
+void FrequentTrafficItems::addMacProtocol(const u_int8_t mac[6], u_int16_t proto_id, u_int32_t value) {
   FrequentTrafficKey_t key;
 
   memset(&key, 0, sizeof(key));

src/HostPools.cpp

@@ -803,7 +803,7 @@ void HostPools::reloadPools() {
 
 /* *************************************** */
 
-bool HostPools::findMacPool(u_int8_t *mac, u_int16_t *found_pool) {
+bool HostPools::findMacPool(const u_int8_t * const mac, u_int16_t *found_pool) {
   VlanAddressTree *cur_tree; /* must use this as tree can be swapped */
   int16_t ret;