Add few scripts to easily update some IPs lists (ntop#1436)

* Add few scripts to easily update some IPs lists

Some IPs lists should be updated frequently: try to easy the process.
The basic idea is taken from d59fefd and a8fe74e (for Azure
addresses): one specific .c.inc file and one script for each protocol.

Add the possibility to don't load a specific list.

Rename the old NDPI_PROTOCOL_HOTMAIL id to NDPI_PROTOCOL_MS_OUTLOOK,
to identify Hotmail/Outlook/Exchange flows.

TODO: ipv6

Remove the 9 addresses associated to BitTorrent: they have been added in
e2f2111 but it is not clear why all the traffic to/from these ips
should be classified as BitTorrent.

* Added quotes

* Added quotes

Co-authored-by: Luca Deri <[email protected]>

src/include/ndpi_protocol_ids.h

@@ -49,7 +49,7 @@ typedef enum {
   NDPI_PROTOCOL_DHCP                  = 18,
   NDPI_PROTOCOL_POSTGRES              = 19,
   NDPI_PROTOCOL_MYSQL                 = 20,
-  NDPI_PROTOCOL_HOTMAIL               = 21,
+  NDPI_PROTOCOL_MS_OUTLOOK            = 21, /* Hotmail / Microsoft Outlook / Exchange */
   NDPI_PROTOCOL_DIRECT_DOWNLOAD_LINK  = 22,
   NDPI_PROTOCOL_MAIL_POPS             = 23,
   NDPI_PROTOCOL_APPLEJUICE            = 24,

src/include/ndpi_typedefs.h

@@ -1386,10 +1386,17 @@ typedef u_int32_t ndpi_init_prefs;
 
 typedef enum
   {
-    ndpi_no_prefs            = 0,
-    ndpi_dont_load_tor_hosts = 1,
-    ndpi_dont_init_libgcrypt = 2,
-    ndpi_enable_ja3_plus     = 4
+    ndpi_no_prefs                  = 0,
+    ndpi_dont_load_tor_list        = (1 << 0),
+    ndpi_dont_init_libgcrypt       = (1 << 1),
+    ndpi_enable_ja3_plus           = (1 << 2),
+    ndpi_dont_load_azure_list      = (1 << 3),
+    ndpi_dont_load_whatsapp_list   = (1 << 4),
+    ndpi_dont_load_amazon_aws_list = (1 << 5),
+    ndpi_dont_load_ethereum_list   = (1 << 6),
+    ndpi_dont_load_zoom_list       = (1 << 7),
+    ndpi_dont_load_cloudflare_list = (1 << 8),
+    ndpi_dont_load_microsoft_list  = (1 << 9),
   } ndpi_prefs;
 
 typedef struct {

src/lib/ndpi_cloudflare_match.c.inc

@@ -0,0 +1,41 @@
+/*
+ *
+ * This file is generated automatically and part of nDPI
+ *
+ * nDPI is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nDPI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/* ****************************************************** */
+
+
+static ndpi_network ndpi_protocol_cloudflare_protocol_list[] = {
+ { 0xADF53000 /* 173.245.48.0/20 */, 20, NDPI_PROTOCOL_CLOUDFLARE },
+ { 0x6715F400 /* 103.21.244.0/22 */, 22, NDPI_PROTOCOL_CLOUDFLARE },
+ { 0x6716C800 /* 103.22.200.0/22 */, 22, NDPI_PROTOCOL_CLOUDFLARE },
+ { 0x671F0400 /* 103.31.4.0/22 */, 22, NDPI_PROTOCOL_CLOUDFLARE },
+ { 0x8D654000 /* 141.101.64.0/18 */, 18, NDPI_PROTOCOL_CLOUDFLARE },
+ { 0x6CA2C000 /* 108.162.192.0/18 */, 18, NDPI_PROTOCOL_CLOUDFLARE },
+ { 0xBE5DF000 /* 190.93.240.0/20 */, 20, NDPI_PROTOCOL_CLOUDFLARE },
+ { 0xBC726000 /* 188.114.96.0/20 */, 20, NDPI_PROTOCOL_CLOUDFLARE },
+ { 0xC5EAF000 /* 197.234.240.0/22 */, 22, NDPI_PROTOCOL_CLOUDFLARE },
+ { 0xC6298000 /* 198.41.128.0/17 */, 17, NDPI_PROTOCOL_CLOUDFLARE },
+ { 0xA29E0000 /* 162.158.0.0/15 */, 15, NDPI_PROTOCOL_CLOUDFLARE },
+ { 0x68100000 /* 104.16.0.0/13 */, 13, NDPI_PROTOCOL_CLOUDFLARE },
+ { 0x68180000 /* 104.24.0.0/14 */, 14, NDPI_PROTOCOL_CLOUDFLARE },
+ { 0xAC400000 /* 172.64.0.0/13 */, 13, NDPI_PROTOCOL_CLOUDFLARE },
+ { 0x83004800 /* 131.0.72.0/22 */, 22, NDPI_PROTOCOL_CLOUDFLARE },
+ /* End */
+ { 0x0, 0, 0 }
+};

src/lib/ndpi_ethereum_match.c.inc

@@ -0,0 +1,51 @@
+/*
+ *
+ * This file is generated automatically and part of nDPI
+ *
+ * nDPI is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nDPI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/* ****************************************************** */
+
+
+static ndpi_network ndpi_protocol_mining_protocol_list[] = {
+ { 0x128A6C43 /* 18.138.108.67/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x03D12D4F /* 3.209.45.79/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x22FF1771 /* 34.255.23.113/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x239EF497 /* 35.158.244.151/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x34BBCF1B /* 52.187.207.27/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0xBFEAA2C6 /* 191.234.162.198/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x34E7A56C /* 52.231.165.108/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x682AD919 /* 104.42.217.25/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x34B0070A /* 52.176.7.10/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x34B0644D /* 52.176.100.77/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x34E8F398 /* 52.232.243.152/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0xC051D0DF /* 192.81.208.223/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x12A8B656 /* 18.168.182.86/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x340E97B1 /* 52.14.151.177/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x34A92A65 /* 52.169.42.101/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x34039EB8 /* 52.3.158.184/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x9F591CD3 /* 159.89.28.211/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x338D4E35 /* 51.141.78.53/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x0D5D3689 /* 13.93.54.137/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x5EED3672 /* 94.237.54.114/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x12DAFA42 /* 18.218.250.66/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x030B9343 /* 3.11.147.67/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x330F74E2 /* 51.15.116.226/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x330F779D /* 51.15.119.157/32 */, 32, NDPI_PROTOCOL_MINING },
+ { 0x330F779D /* 51.15.119.157/32 */, 32, NDPI_PROTOCOL_MINING },
+ /* End */
+ { 0x0, 0, 0 }
+};

src/lib/ndpi_main.c

@@ -47,6 +47,16 @@
 
 #include "ndpi_content_match.c.inc"
 #include "ndpi_azure_match.c.inc"
+#include "ndpi_tor_match.c.inc"
+#include "ndpi_whatsapp_match.c.inc"
+#include "ndpi_amazon_aws_match.c.inc"
+#include "ndpi_ethereum_match.c.inc"
+#include "ndpi_zoom_match.c.inc"
+#include "ndpi_cloudflare_match.c.inc"
+#include "ndpi_ms_office365_match.c.inc"
+#include "ndpi_ms_onedrive_match.c.inc"
+#include "ndpi_ms_outlook_match.c.inc"
+#include "ndpi_ms_skype_teams_match.c.inc"
 #include "third_party/include/ndpi_patricia.h"
 #include "third_party/include/ndpi_md5.h"
 
@@ -2112,17 +2122,13 @@ int ndpi_load_ipv4_ptree(struct ndpi_detection_module_struct *ndpi_str,
 /* ******************************************* */
 
 static void ndpi_init_ptree_ipv4(struct ndpi_detection_module_struct *ndpi_str,
-				 void *ptree, ndpi_network host_list[],
-                                 u_int8_t skip_tor_hosts) {
+				 void *ptree, ndpi_network host_list[]) {
   int i;
 
   for(i = 0; host_list[i].network != 0x0; i++) {
     struct in_addr pin;
     ndpi_patricia_node_t *node;
 
-    if(skip_tor_hosts && (host_list[i].value == NDPI_PROTOCOL_TOR))
-      continue;
-
     pin.s_addr = htonl(host_list[i].network);
     if((node = add_to_ptree(ptree, AF_INET, &pin, host_list[i].cidr /* bits */)) != NULL) {
       node->value.u.uv32.user_value = host_list[i].value, node->value.u.uv32.additional_user_value = 0;
@@ -2383,9 +2389,27 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
 #endif
 
   if((ndpi_str->protocols_ptree = ndpi_patricia_new(32 /* IPv4 */)) != NULL) {
-    ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, host_protocol_list, prefs & ndpi_dont_load_tor_hosts);
-    ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_microsoft_azure_protocol_list,
-			 prefs & ndpi_dont_load_tor_hosts); /* Microsoft Azure */
+    ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, host_protocol_list);
+    if(!(prefs & ndpi_dont_load_tor_list))
+      ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_tor_protocol_list);
+    if(!(prefs & ndpi_dont_load_azure_list))
+      ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_microsoft_azure_protocol_list);
+    if(!(prefs & ndpi_dont_load_whatsapp_list))
+      ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_whatsapp_protocol_list);
+    if(!(prefs & ndpi_dont_load_amazon_aws_list))
+      ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_amazon_aws_protocol_list);
+    if(!(prefs & ndpi_dont_load_ethereum_list))
+      ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_mining_protocol_list);
+    if(!(prefs & ndpi_dont_load_zoom_list))
+      ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_zoom_protocol_list);
+    if(!(prefs & ndpi_dont_load_cloudflare_list))
+      ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_cloudflare_protocol_list);
+    if(!(prefs & ndpi_dont_load_microsoft_list)) {
+      ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_microsoft_365_protocol_list);
+      ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_ms_one_drive_protocol_list);
+      ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_ms_outlook_protocol_list);
+      ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_skype_teams_protocol_list);
+    }
   }
 
   ndpi_str->ip_risk_mask_ptree = ndpi_patricia_new(32 /* IPv4 */);

src/lib/ndpi_ms_office365_match.c.inc

@@ -0,0 +1,38 @@
+/*
+ *
+ * This file is generated automatically and part of nDPI
+ *
+ * nDPI is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nDPI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/* ****************************************************** */
+
+
+static ndpi_network ndpi_protocol_microsoft_365_protocol_list[] = {
+ { 0x0D6B8C06 /* 13.107.140.6/32 */, 32, NDPI_PROTOCOL_MICROSOFT_365 },
+ { 0x0D6B120F /* 13.107.18.15/32 */, 32, NDPI_PROTOCOL_MICROSOFT_365 },
+ { 0x0D6B06AB /* 13.107.6.171/32 */, 32, NDPI_PROTOCOL_MICROSOFT_365 },
+ { 0x14BE8000 /* 20.190.128.0/18 */, 18, NDPI_PROTOCOL_MICROSOFT_365 },
+ { 0x287E0000 /* 40.126.0.0/18 */, 18, NDPI_PROTOCOL_MICROSOFT_365 },
+ { 0x346C0000 /* 52.108.0.0/14 */, 14, NDPI_PROTOCOL_MICROSOFT_365 },
+ { 0x34EE6A74 /* 52.238.106.116/32 */, 32, NDPI_PROTOCOL_MICROSOFT_365 },
+ { 0x34F4CB48 /* 52.244.203.72/32 */, 32, NDPI_PROTOCOL_MICROSOFT_365 },
+ { 0x34F4CFAC /* 52.244.207.172/32 */, 32, NDPI_PROTOCOL_MICROSOFT_365 },
+ { 0x34F4DFC6 /* 52.244.223.198/32 */, 32, NDPI_PROTOCOL_MICROSOFT_365 },
+ { 0x34F425A8 /* 52.244.37.168/32 */, 32, NDPI_PROTOCOL_MICROSOFT_365 },
+ { 0x34F796BF /* 52.247.150.191/32 */, 32, NDPI_PROTOCOL_MICROSOFT_365 },
+ /* End */
+ { 0x0, 0, 0 }
+};

src/lib/ndpi_ms_onedrive_match.c.inc

@@ -0,0 +1,31 @@
+/*
+ *
+ * This file is generated automatically and part of nDPI
+ *
+ * nDPI is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nDPI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/* ****************************************************** */
+
+
+static ndpi_network ndpi_protocol_ms_one_drive_protocol_list[] = {
+ { 0x68928000 /* 104.146.128.0/17 */, 17, NDPI_PROTOCOL_MS_ONE_DRIVE },
+ { 0x0D6B8800 /* 13.107.136.0/22 */, 22, NDPI_PROTOCOL_MS_ONE_DRIVE },
+ { 0x96AB2800 /* 150.171.40.0/22 */, 22, NDPI_PROTOCOL_MS_ONE_DRIVE },
+ { 0x286C8000 /* 40.108.128.0/17 */, 17, NDPI_PROTOCOL_MS_ONE_DRIVE },
+ { 0x34680000 /* 52.104.0.0/14 */, 14, NDPI_PROTOCOL_MS_ONE_DRIVE },
+ /* End */
+ { 0x0, 0, 0 }
+};

src/lib/ndpi_ms_outlook_match.c.inc

@@ -0,0 +1,42 @@
+/*
+ *
+ * This file is generated automatically and part of nDPI
+ *
+ * nDPI is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nDPI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/* ****************************************************** */
+
+
+static ndpi_network ndpi_protocol_ms_outlook_protocol_list[] = {
+ { 0x682F0000 /* 104.47.0.0/17 */, 17, NDPI_PROTOCOL_MS_OUTLOOK },
+ { 0x0D6B8000 /* 13.107.128.0/22 */, 22, NDPI_PROTOCOL_MS_OUTLOOK },
+ { 0x0D6B120A /* 13.107.18.10/31 */, 31, NDPI_PROTOCOL_MS_OUTLOOK },
+ { 0x0D6B0698 /* 13.107.6.152/31 */, 31, NDPI_PROTOCOL_MS_OUTLOOK },
+ { 0x83FD21D7 /* 131.253.33.215/32 */, 32, NDPI_PROTOCOL_MS_OUTLOOK },
+ { 0x84F50000 /* 132.245.0.0/16 */, 16, NDPI_PROTOCOL_MS_OUTLOOK },
+ { 0x96AB2000 /* 150.171.32.0/22 */, 22, NDPI_PROTOCOL_MS_OUTLOOK },
+ { 0xCC4FC5D7 /* 204.79.197.215/32 */, 32, NDPI_PROTOCOL_MS_OUTLOOK },
+ { 0x1767A000 /* 23.103.160.0/20 */, 20, NDPI_PROTOCOL_MS_OUTLOOK },
+ { 0x28680000 /* 40.104.0.0/15 */, 15, NDPI_PROTOCOL_MS_OUTLOOK },
+ { 0x286B0000 /* 40.107.0.0/16 */, 16, NDPI_PROTOCOL_MS_OUTLOOK },
+ { 0x285C0000 /* 40.92.0.0/15 */, 15, NDPI_PROTOCOL_MS_OUTLOOK },
+ { 0x28600000 /* 40.96.0.0/13 */, 13, NDPI_PROTOCOL_MS_OUTLOOK },
+ { 0x34640000 /* 52.100.0.0/14 */, 14, NDPI_PROTOCOL_MS_OUTLOOK },
+ { 0x34EE4E58 /* 52.238.78.88/32 */, 32, NDPI_PROTOCOL_MS_OUTLOOK },
+ { 0x34600000 /* 52.96.0.0/14 */, 14, NDPI_PROTOCOL_MS_OUTLOOK },
+ /* End */
+ { 0x0, 0, 0 }
+};

src/lib/ndpi_ms_skype_teams_match.c.inc

@@ -0,0 +1,31 @@
+/*
+ *
+ * This file is generated automatically and part of nDPI
+ *
+ * nDPI is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nDPI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/* ****************************************************** */
+
+
+static ndpi_network ndpi_protocol_skype_teams_protocol_list[] = {
+ { 0x0D6B4000 /* 13.107.64.0/18 */, 18, NDPI_PROTOCOL_SKYPE_TEAMS },
+ { 0x34700000 /* 52.112.0.0/14 */, 14, NDPI_PROTOCOL_SKYPE_TEAMS },
+ { 0x34780000 /* 52.120.0.0/14 */, 14, NDPI_PROTOCOL_SKYPE_TEAMS },
+ { 0x34EE778D /* 52.238.119.141/32 */, 32, NDPI_PROTOCOL_SKYPE_TEAMS },
+ { 0x34F4A0CF /* 52.244.160.207/32 */, 32, NDPI_PROTOCOL_SKYPE_TEAMS },
+ /* End */
+ { 0x0, 0, 0 }
+};