Merge pull request #3 from ntop/dev

sync

httpdocs/js/ntopng_utils.js

@@ -540,6 +540,15 @@ function makeFindHostBeforeSubmitCallback(http_prefix) {
   return function(form, data) {
     if (data.type == "mac") {
       form.attr("action", http_prefix + "/lua/mac_details.lua");
+    } else if (data.type == "network") {
+      form.attr("action", http_prefix + "/lua/hosts_stats.lua");
+      /* Must add also the network to properly set the destination link */
+      $('<input>').attr({
+        type: 'hidden',
+        id: 'network',
+        name: 'network',
+        value: data.network,
+      }).appendTo(form);
     } else if (data.type == "snmp") {
       form.attr("action", http_prefix + "/lua/pro/enterprise/snmp_interface_details.lua");
       /* Must add also the snmp port index to properly set the destination link */

include/Flow.h

@@ -39,19 +39,6 @@ typedef struct {
   InterarrivalStats pktTime;
 } FlowPacketStats;
 
-typedef enum {
-  SSL_STAGE_UNKNOWN = 0,
-  SSL_STAGE_HELLO,
-  SSL_STAGE_CCS
-} FlowSSLStage;
-
-typedef enum {
-  SSL_ENCRYPTION_PLAIN = 0x0,
-  SSL_ENCRYPTION_SERVER = 0x1,
-  SSL_ENCRYPTION_CLIENT = 0x2,
-  SSL_ENCRYPTION_BOTH = 0x3,
-} FlowSSLEncryptionStatus;
-
 class Flow : public GenericHashEntry {
  private:
   Host *cli_host, *srv_host;
@@ -107,13 +94,10 @@ class Flow : public GenericHashEntry {
 
     struct {
       char *certificate, *server_certificate;
-      FlowSSLStage cli_stage, srv_stage;
-      u_int8_t hs_packets;
-      bool is_data;
-      /* firstdata refers to the time where encryption is active on both ends */
-      bool firstdata_seen;
-      struct timeval clienthello_time, hs_end_time, lastdata_time;
-      float hs_delta_time, delta_firstData, deltaTime_data;
+      /* Certificate dissection */
+      char *certificate_buf_leftover;
+      u_int certificate_leftover;
+      bool dissect_certificate;
     } ssl;
 
     struct {
@@ -277,6 +261,7 @@ class Flow : public GenericHashEntry {
   inline char* getBitTorrentHash() { return(bt_hash);          };
   inline void  setBTHash(char *h)  { if(!h) return; if(bt_hash) { free(bt_hash); bt_hash = NULL; }; bt_hash = strdup(h); }
   inline void  setServerName(char *v)  { if(host_server_name) free(host_server_name);  host_server_name = strdup(v); }
+  void setTcpFlags(u_int8_t flags, bool src2dst_direction);
   void updateTcpFlags(const struct bpf_timeval *when,
 		      u_int8_t flags, bool src2dst_direction);
   void incTcpBadStats(bool src2dst_direction,
@@ -395,8 +380,8 @@ class Flow : public GenericHashEntry {
   void guessProtocol();
   bool dumpFlow(bool dump_alert);
   bool match(AddressTree *ptree);
-  void dissectSSL(u_int8_t *payload, u_int16_t payload_len, const struct bpf_timeval *when, bool cli2srv);
   void dissectHTTP(bool src2dst_direction, char *payload, u_int16_t payload_len);
+  void dissectSSL(char *payload, u_int16_t payload_len);
   void dissectSSDP(bool src2dst_direction, char *payload, u_int16_t payload_len);
   void dissectMDNS(u_int8_t *payload, u_int16_t payload_len);
   void dissectBittorrent(char *payload, u_int16_t payload_len);
@@ -420,10 +405,6 @@ class Flow : public GenericHashEntry {
   inline char* getHTTPContentType() { return(isHTTP() ? protos.http.last_content_type : (char*)"");   }
   inline char* getSSLCertificate()  { return(isSSL() ? protos.ssl.certificate : (char*)""); }
   bool isSSLProto();
-  inline bool isSSLData()              { return(isSSLProto() && good_ssl_hs && protos.ssl.is_data);  }
-  inline bool isSSLHandshake()         { return(isSSLProto() && good_ssl_hs && !protos.ssl.is_data); }
-  inline bool hasSSLHandshakeEnded()   { return(getSSLEncryptionStatus() == SSL_ENCRYPTION_BOTH);    }
-  FlowSSLEncryptionStatus getSSLEncryptionStatus();
 
 #if defined(NTOPNG_PRO) && !defined(HAVE_NEDGE)
   inline void updateProfile()     { trafficProfile = iface->getFlowProfile(this); }
@@ -455,9 +436,19 @@ class Flow : public GenericHashEntry {
   inline void      setVRFid(u_int32_t v)  { vrfId = v;                              }
 
   inline void setFlowNwLatency(const struct timeval * const tv, bool client) {
-    if(client) memcpy(&clientNwLatency, tv, sizeof(*tv));
-    else memcpy(&serverNwLatency, tv, sizeof(*tv));
-  };
+    if(client) {
+      memcpy(&clientNwLatency, tv, sizeof(*tv));
+      if(cli_host) cli_host->updateRoundTripTime(Utils::timeval2ms(&clientNwLatency));
+    } else {
+      memcpy(&serverNwLatency, tv, sizeof(*tv));
+      if(srv_host) srv_host->updateRoundTripTime(Utils::timeval2ms(&serverNwLatency));
+    }
+  }
+  inline void setRtt() {
+    rttSec = ((float)(serverNwLatency.tv_sec + clientNwLatency.tv_sec))
+      +((float)(serverNwLatency.tv_usec + clientNwLatency.tv_usec)) / (float)1000000;
+  }
+  inline void setFlowApplLatency(float latency_msecs) { applLatencyMsec = latency_msecs; }
   inline bool      setFlowDevice(u_int32_t device_ip, u_int16_t inidx, u_int16_t outidx) {
     if((flow_device.device_ip > 0 && flow_device.device_ip != device_ip)
        || (flow_device.in_index > 0 && flow_device.in_index != inidx)

include/Host.h

@@ -221,7 +221,7 @@ class Host : public GenericHashEntry {
   virtual void  serialize2redis() {};
   bool addIfMatching(lua_State* vm, AddressTree * ptree, char *key);
   bool addIfMatching(lua_State* vm, u_int8_t *mac);
-  void updateSynFlags(time_t when, u_int8_t flags, Flow *f, bool syn_sent);
+  void updateSynAlertsCounter(time_t when, u_int8_t flags, Flow *f, bool syn_sent);
   inline void updateRoundTripTime(u_int32_t rtt_msecs) {
     if(as) as->updateRoundTripTime(rtt_msecs);
   }

include/ntop_flow.h

@@ -158,11 +158,13 @@
 #define DST_AS_MAP  NTOP_BASE_ID+444
 #define NPROBE_IPV4_ADDRESS  NTOP_BASE_ID+471 /* Exported only via ZMQ */
 /* NTOP_BASE_ID+82 ->87  Available (deprecated) */
-#define NUM_PKTS_UP_TO_128_BYTES   NTOP_BASE_ID+88
+#define CLIENT_TCP_FLAGS            NTOP_BASE_ID+78
+#define SERVER_TCP_FLAGS            NTOP_BASE_ID+79
+#define NUM_PKTS_UP_TO_128_BYTES    NTOP_BASE_ID+88
 #define NUM_PKTS_128_TO_256_BYTES   NTOP_BASE_ID+89
 #define NUM_PKTS_256_TO_512_BYTES   NTOP_BASE_ID+90
-#define NUM_PKTS_512_TO_1024_BYTES   NTOP_BASE_ID+91
-#define NUM_PKTS_1024_TO_1514_BYTES   NTOP_BASE_ID+92
+#define NUM_PKTS_512_TO_1024_BYTES  NTOP_BASE_ID+91
+#define NUM_PKTS_1024_TO_1514_BYTES NTOP_BASE_ID+92
 #define NUM_PKTS_OVER_1514_BYTES   NTOP_BASE_ID+93
 #define CUMULATIVE_ICMP_TYPE NTOP_BASE_ID+98
 #define SRC_IP_COUNTRY   NTOP_BASE_ID+101

include/ntop_typedefs.h

@@ -189,10 +189,11 @@ typedef struct zmq_flow_core {
   u_int16_t src_port, dst_port, inIndex, outIndex;
   ndpi_proto l7_proto;
   u_int16_t vlan_id, pkt_sampling_rate;
-  u_int8_t l4_proto, tcp_flags;
+  u_int8_t l4_proto;
   u_int32_t in_pkts, in_bytes, out_pkts, out_bytes, vrfId;
   u_int8_t absolute_packet_octet_counters;
   struct {
+    u_int8_t tcp_flags, client_tcp_flags, server_tcp_flags;
     u_int32_t ooo_in_pkts, ooo_out_pkts;
     u_int32_t retr_in_pkts, retr_out_pkts;
     u_int32_t lost_in_pkts, lost_out_pkts;

scripts/lua/find_host.lua

@@ -41,6 +41,26 @@ for mac, name in pairs(mac_to_name) do
    end
 end
 
+-- Look by network
+local network_stats = interface.getNetworksStats()
+
+for network, stats in pairs(network_stats) do
+   local name = getFullLocalNetworkName(network)
+
+   if string.contains(string.lower(name), string.lower(query)) then
+      local network_id = stats.network_id
+
+      results[#results + 1] = {
+	 name = name,
+	 type="network", network = network_id,
+      }
+
+      if #results >= max_num_to_find then
+	 break
+      end
+   end
+end
+
 -- Check also in the mac addresses of snmp devices
 -- The query can be partial so we can't use functions to
 -- test if it'a an IPv4, an IPv6, or a mac as they would yield

scripts/lua/host_details.lua

@@ -1930,6 +1930,11 @@ drawGraphs(ifId, schema, tags, _GET["zoom"], url, selected_epoch, {
       {schema="host:unreachable_flows",      label=i18n("graphs.total_unreachable_flows")},
       {schema="host:contacts",               label=i18n("graphs.active_host_contacts")},
       {schema="host:total_alerts",           label=i18n("details.alerts")},
+      {schema="host:dns_pkts_rcvd",          label="DNS Packets Rcvd"},
+      {schema="host:dns_pkts_sent",          label="DNS Packets Sent"},
+      {schema="host:arp_requests_pkts",      label="ARP Requests Packets"},
+      {schema="host:udp_pkts",               label="UDP Packets"},
+      {schema="host:tcp_pkts_stats",         label="TCP Packets Stats"},
 
       {schema="host:1d_delta_traffic_volume",  label="1 Day Traffic Delta"}, -- TODO localize
       {schema="host:1d_delta_flows",           label="1 Day Active Flows Delta"}, -- TODO localize

scripts/lua/host_pkt_distro.lua

@@ -2,7 +2,7 @@
 -- (C) 2013-18 - ntop.org
 --
 
-dirs = ntop.getDirs()
+local dirs = ntop.getDirs()
 package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path
 
 require "lua_utils"
@@ -11,55 +11,55 @@ sendHTTPContentTypeHeader('text/html')
 
 interface.select(ifname)
 
-mode = _GET["direction"]
-type = _GET["distr"]
-host_info = url2hostinfo(_GET)
-host = interface.getHostInfo(host_info["host"],host_info["vlan"])
+local mode = _GET["direction"]
+local type = _GET["distr"]
+local host_info = url2hostinfo(_GET)
+local host = interface.getHostInfo(host_info["host"],host_info["vlan"])
 
 
 if(host == nil) then
    print("<div class=\"alert alert-danger\"><img src=".. ntop.getHttpPrefix() .. "/img/warning.png> This flow cannot be found (expired ?)</div>")
 else
 
-if((type == nil) or (type == "size")) then
-
-   if((mode == nil) or (mode == "sent")) then
-
-      what = host["pktStats.sent"]
-
-   else
-
-      what = host["pktStats.recv"]
+   if((type == nil) or (type == "size")) then
+
+      if((mode == nil) or (mode == "sent")) then
+
+	 what = host["pktStats.sent"]
+
+      else
+
+	 what = host["pktStats.recv"]
+      end
    end
-end
 
-tot = 0
-for key, value in pairs(what) do
-   tot = tot + value
-end
+   local tot = 0
+   for key, value in pairs(what) do
+      tot = tot + value
+   end
+
+   local threshold = (5 * tot) / 100
 
-threshold = (5 * tot) / 100
+   print "[\n"
+   local num = 0
+   local s = 0
+   for key, value in pairs(what) do
+      if(value > threshold) then
+	 if(num > 0) then
+	    print ",\n"
+	 end
 
-print "[\n"
-num = 0
-s = 0
-for key, value in pairs(what) do
-   if(value > threshold) then
-      if(num > 0) then
-	 print ",\n"
+	 print("\t { \"label\": \"" .. key .."\", \"value\": ".. value .." }")
+	 num = num + 1
+	 s = s + value
       end
-
-      print("\t { \"label\": \"" .. key .."\", \"value\": ".. value .." }") 
-      num = num + 1
-      s = s + value
    end
-end
 
-if(tot > s) then
-   print(",\t { \"label\": \"Other\", \"value\": ".. (tot-s) .." }") 
-end
+   if(tot > s) then
+      print(",\t { \"label\": \"Other\", \"value\": ".. (tot-s) .." }")
+   end
 
 
-print "\n]"
+   print "\n]"
 
 end

scripts/lua/modules/alert_utils.lua

@@ -602,7 +602,7 @@ local function formatAlertEntity(ifid, entity_type, entity_value, entity_info)
 	  "'>"..value.."</a>"
       end
    elseif entity_type == "network" then
-      value = hostkey2hostinfo(entity_value)["host"]
+      value = getLocalNetworkAlias(hostkey2hostinfo(entity_value)["host"])
 
       if entity_info ~= nil then
 	 value = "<a href='"..ntop.getHttpPrefix().."/lua/network_details.lua?network="..

scripts/lua/modules/flow_utils.lua

@@ -2042,28 +2042,26 @@ function printActiveFlowsDropdown(base_url, page_params, ifstats, ndpistats)
        </div>\
     ']]
 
-    if interface.isPacketInterface() then
-       -- TCP flow state filter
-       local tcp_state_params = table.clone(page_params)
-       tcp_state_params["tcp_flow_state"] = nil
+    -- TCP flow state filter
+    local tcp_state_params = table.clone(page_params)
+    tcp_state_params["tcp_flow_state"] = nil
 
-       print[[, '\
+    print[[, '\
        <div class="btn-group">\
 	  <button class="btn btn-link dropdown-toggle" data-toggle="dropdown">]] print(i18n("flows_page.tcp_state")) print(getParamFilter(page_params, "tcp_flow_state")) print[[<span class="caret"></span></button>\
 	  <ul class="dropdown-menu" role="menu">\
 	  <li><a href="]] print(getPageUrl(base_url, tcp_state_params)) print[[">]] print(i18n("flows_page.all_flows")) print[[</a></li>\]]
 
-       local entries = {}
-       for _, entry in pairs({"established", "connecting", "closed", "reset"}) do
-	  entries[#entries + 1] = {entry, tcp_flow_state_utils.state2i18n(entry)}
-       end
+    local entries = {}
+    for _, entry in pairs({"established", "connecting", "closed", "reset"}) do
+       entries[#entries + 1] = {entry, tcp_flow_state_utils.state2i18n(entry)}
+    end
 
-       printDropdownEntries(entries, base_url, tcp_state_params, "tcp_flow_state", page_params.tcp_flow_state)
-       print[[\
+    printDropdownEntries(entries, base_url, tcp_state_params, "tcp_flow_state", page_params.tcp_flow_state)
+    print[[\
 	  </ul>\
        </div>\
     ']]
-    end
 
     -- Unidirectional flows selector
     local traffic_type_params = table.clone(page_params)

scripts/lua/modules/lua_utils.lua

@@ -1573,14 +1573,14 @@ end
 
 -- ##############################################
 
-function getLocalNetworkAliassKey()
+function getLocalNetworkAliasKey()
    return "ntopng.network_aliases"
 end
 
 -- ##############################################
 
 function getLocalNetworkAlias(network)
-   local alias = ntop.getHashCache(getLocalNetworkAliassKey(), network)
+   local alias = ntop.getHashCache(getLocalNetworkAliasKey(), network)
 
    if not isEmptyString(alias) then
       return alias
@@ -1605,9 +1605,9 @@ end
 
 function setLocalNetworkAlias(network, alias)
    if((network ~= alias) or isEmptyString(alias)) then
-      ntop.setHashCache(getLocalNetworkAliassKey(), network, alias)
+      ntop.setHashCache(getLocalNetworkAliasKey(), network, alias)
    else
-      ntop.delHashCache(getLocalNetworkAliassKey(), network)
+      ntop.delHashCache(getLocalNetworkAliasKey(), network)
    end
 end