diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 2e8c76d2601..943de51187e 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -1704,6 +1704,7 @@ typedef enum {
        correct detection/classification.
        See #1946 for other details */
     ndpi_enable_tcp_ack_payload_heuristic = (1 << 17),
+    ndpi_dont_load_crawlers_list = (1 << 18),
   } ndpi_prefs;
 
 typedef struct {
diff --git a/src/lib/inc_generated/ndpi_crawlers_match.c.inc b/src/lib/inc_generated/ndpi_crawlers_match.c.inc
new file mode 100644
index 00000000000..c25c784f53f
--- /dev/null
+++ b/src/lib/inc_generated/ndpi_crawlers_match.c.inc
@@ -0,0 +1,592 @@
+/*
+ *
+ * This file is generated automatically and part of nDPI
+ *
+ * nDPI is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nDPI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/* ****************************************************** */
+
+
+static ndpi_network ndpi_http_crawler_bot_protocol_list[] = {
+ { 0xC0B20500 /* 192.178.5.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x2264B660 /* 34.100.182.96/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22653290 /* 34.101.50.144/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x2276FE00 /* 34.118.254.0/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22764200 /* 34.118.66.0/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x227EB260 /* 34.126.178.96/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22929690 /* 34.146.150.144/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22936E90 /* 34.147.110.144/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22974A90 /* 34.151.74.144/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22983240 /* 34.152.50.64/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x229A7290 /* 34.154.114.144/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x229B6220 /* 34.155.98.32/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22A512B0 /* 34.165.18.176/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22AFA040 /* 34.175.160.64/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22B08210 /* 34.176.130.16/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22165500 /* 34.22.85.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22405240 /* 34.64.82.64/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x2241F270 /* 34.65.242.112/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22503250 /* 34.80.50.80/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x2258C200 /* 34.88.194.0/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22590A50 /* 34.89.10.80/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x2259C650 /* 34.89.198.80/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x2260A230 /* 34.96.162.48/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F7F3F0 /* 35.247.243.240/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94000 /* 66.249.64.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94080 /* 66.249.64.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F940A0 /* 66.249.64.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F940C0 /* 66.249.64.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F940E0 /* 66.249.64.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94020 /* 66.249.64.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94040 /* 66.249.64.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94060 /* 66.249.64.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94100 /* 66.249.65.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94180 /* 66.249.65.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F941A0 /* 66.249.65.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F941C0 /* 66.249.65.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F941E0 /* 66.249.65.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94120 /* 66.249.65.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94140 /* 66.249.65.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94160 /* 66.249.65.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94200 /* 66.249.66.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94280 /* 66.249.66.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F942C0 /* 66.249.66.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94220 /* 66.249.66.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94240 /* 66.249.66.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94260 /* 66.249.66.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94400 /* 66.249.68.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94420 /* 66.249.68.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94440 /* 66.249.68.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94500 /* 66.249.69.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94580 /* 66.249.69.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F945A0 /* 66.249.69.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F945C0 /* 66.249.69.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F945E0 /* 66.249.69.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94520 /* 66.249.69.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94540 /* 66.249.69.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94560 /* 66.249.69.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94600 /* 66.249.70.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94680 /* 66.249.70.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F946A0 /* 66.249.70.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F946C0 /* 66.249.70.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F946E0 /* 66.249.70.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94620 /* 66.249.70.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94640 /* 66.249.70.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94660 /* 66.249.70.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94700 /* 66.249.71.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94780 /* 66.249.71.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F947A0 /* 66.249.71.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F947C0 /* 66.249.71.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F947E0 /* 66.249.71.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94720 /* 66.249.71.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94740 /* 66.249.71.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94760 /* 66.249.71.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94800 /* 66.249.72.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94880 /* 66.249.72.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F948A0 /* 66.249.72.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F948C0 /* 66.249.72.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F948E0 /* 66.249.72.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94820 /* 66.249.72.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94840 /* 66.249.72.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94860 /* 66.249.72.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94900 /* 66.249.73.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94980 /* 66.249.73.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F949A0 /* 66.249.73.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F949C0 /* 66.249.73.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F949E0 /* 66.249.73.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94920 /* 66.249.73.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94940 /* 66.249.73.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94960 /* 66.249.73.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94A00 /* 66.249.74.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94A80 /* 66.249.74.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94A20 /* 66.249.74.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94A40 /* 66.249.74.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94A60 /* 66.249.74.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94B00 /* 66.249.75.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94B80 /* 66.249.75.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94BA0 /* 66.249.75.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94BC0 /* 66.249.75.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94BE0 /* 66.249.75.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94B20 /* 66.249.75.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94B40 /* 66.249.75.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94B60 /* 66.249.75.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94C00 /* 66.249.76.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94C80 /* 66.249.76.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94CA0 /* 66.249.76.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94CC0 /* 66.249.76.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94CE0 /* 66.249.76.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94C20 /* 66.249.76.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94C40 /* 66.249.76.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94C60 /* 66.249.76.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94D00 /* 66.249.77.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94D80 /* 66.249.77.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94DA0 /* 66.249.77.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94DC0 /* 66.249.77.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94D20 /* 66.249.77.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94D40 /* 66.249.77.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94D60 /* 66.249.77.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94F00 /* 66.249.79.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94F80 /* 66.249.79.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94FA0 /* 66.249.79.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94FC0 /* 66.249.79.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94FE0 /* 66.249.79.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94F20 /* 66.249.79.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94F40 /* 66.249.79.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F94F60 /* 66.249.79.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0xD155EE00 /* 209.85.238.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0xD155EE80 /* 209.85.238.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0xD155EEA0 /* 209.85.238.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0xD155EEC0 /* 209.85.238.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0xD155EEE0 /* 209.85.238.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0xD155EE20 /* 209.85.238.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0xD155EE40 /* 209.85.238.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0xD155EE60 /* 209.85.238.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95700 /* 66.249.87.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95780 /* 66.249.87.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F957A0 /* 66.249.87.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F957C0 /* 66.249.87.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F957E0 /* 66.249.87.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95720 /* 66.249.87.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95740 /* 66.249.87.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95760 /* 66.249.87.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95900 /* 66.249.89.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95980 /* 66.249.89.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F959A0 /* 66.249.89.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F959C0 /* 66.249.89.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F959E0 /* 66.249.89.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95920 /* 66.249.89.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95940 /* 66.249.89.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95960 /* 66.249.89.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95A00 /* 66.249.90.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95A80 /* 66.249.90.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95AA0 /* 66.249.90.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95AC0 /* 66.249.90.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95AE0 /* 66.249.90.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95A20 /* 66.249.90.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95A40 /* 66.249.90.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95A60 /* 66.249.90.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95B00 /* 66.249.91.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95B80 /* 66.249.91.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95BA0 /* 66.249.91.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95BC0 /* 66.249.91.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95BE0 /* 66.249.91.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95B20 /* 66.249.91.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95B40 /* 66.249.91.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95B60 /* 66.249.91.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95C00 /* 66.249.92.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95C80 /* 66.249.92.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95CC0 /* 66.249.92.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95C20 /* 66.249.92.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x42F95C40 /* 66.249.92.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x480EC700 /* 72.14.199.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x480EC780 /* 72.14.199.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x480EC7A0 /* 72.14.199.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x480EC7C0 /* 72.14.199.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x480EC7E0 /* 72.14.199.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x480EC720 /* 72.14.199.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x480EC740 /* 72.14.199.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x480EC760 /* 72.14.199.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D9400 /* 74.125.148.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D9480 /* 74.125.148.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D94A0 /* 74.125.148.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D94C0 /* 74.125.148.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D94E0 /* 74.125.148.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D9420 /* 74.125.148.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D9440 /* 74.125.148.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D9460 /* 74.125.148.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D9500 /* 74.125.149.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D9580 /* 74.125.149.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D95A0 /* 74.125.149.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D95C0 /* 74.125.149.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D9520 /* 74.125.149.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D9540 /* 74.125.149.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D9560 /* 74.125.149.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D9600 /* 74.125.150.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D9620 /* 74.125.150.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D9640 /* 74.125.150.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D9700 /* 74.125.151.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D9780 /* 74.125.151.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D97A0 /* 74.125.151.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D97C0 /* 74.125.151.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D97E0 /* 74.125.151.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D9720 /* 74.125.151.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D9740 /* 74.125.151.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7D9760 /* 74.125.151.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DD800 /* 74.125.216.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DD880 /* 74.125.216.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DD8A0 /* 74.125.216.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DD8C0 /* 74.125.216.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DD8E0 /* 74.125.216.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DD820 /* 74.125.216.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DD840 /* 74.125.216.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DD860 /* 74.125.216.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DD900 /* 74.125.217.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DD980 /* 74.125.217.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DD920 /* 74.125.217.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DD940 /* 74.125.217.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DD960 /* 74.125.217.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DDA00 /* 74.125.218.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DDA80 /* 74.125.218.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DDAA0 /* 74.125.218.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DDAC0 /* 74.125.218.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DDA20 /* 74.125.218.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DDA40 /* 74.125.218.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4A7DDA60 /* 74.125.218.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C000 /* 107.178.192.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C080 /* 107.178.192.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C020 /* 107.178.192.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C040 /* 107.178.192.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C060 /* 107.178.192.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C100 /* 107.178.193.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C180 /* 107.178.193.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C1A0 /* 107.178.193.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C1C0 /* 107.178.193.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C1E0 /* 107.178.193.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C120 /* 107.178.193.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C140 /* 107.178.193.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C160 /* 107.178.193.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C200 /* 107.178.194.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C280 /* 107.178.194.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C2A0 /* 107.178.194.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C2C0 /* 107.178.194.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C2E0 /* 107.178.194.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C220 /* 107.178.194.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C240 /* 107.178.194.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C260 /* 107.178.194.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C300 /* 107.178.195.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C380 /* 107.178.195.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C3A0 /* 107.178.195.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C3C0 /* 107.178.195.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C3E0 /* 107.178.195.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C320 /* 107.178.195.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C340 /* 107.178.195.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C360 /* 107.178.195.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C400 /* 107.178.196.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C480 /* 107.178.196.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C4A0 /* 107.178.196.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C4C0 /* 107.178.196.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C4E0 /* 107.178.196.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C420 /* 107.178.196.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C440 /* 107.178.196.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C460 /* 107.178.196.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C500 /* 107.178.197.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C580 /* 107.178.197.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C5A0 /* 107.178.197.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C5C0 /* 107.178.197.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C5E0 /* 107.178.197.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C520 /* 107.178.197.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C540 /* 107.178.197.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C560 /* 107.178.197.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C600 /* 107.178.198.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C680 /* 107.178.198.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C6A0 /* 107.178.198.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C6C0 /* 107.178.198.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C620 /* 107.178.198.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C640 /* 107.178.198.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C660 /* 107.178.198.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C700 /* 107.178.199.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C780 /* 107.178.199.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C7A0 /* 107.178.199.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C7C0 /* 107.178.199.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C7E0 /* 107.178.199.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C720 /* 107.178.199.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C740 /* 107.178.199.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2C760 /* 107.178.199.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2CA00 /* 107.178.202.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2CA20 /* 107.178.202.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2CA40 /* 107.178.202.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2CA60 /* 107.178.202.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2CB00 /* 107.178.203.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2CB80 /* 107.178.203.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2CBA0 /* 107.178.203.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2CBC0 /* 107.178.203.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2CBE0 /* 107.178.203.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2CB20 /* 107.178.203.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2CB40 /* 107.178.203.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2CB60 /* 107.178.203.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2E000 /* 107.178.224.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2E080 /* 107.178.224.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2E0A0 /* 107.178.224.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2E0C0 /* 107.178.224.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2E0E0 /* 107.178.224.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2E020 /* 107.178.224.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2E040 /* 107.178.224.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2E060 /* 107.178.224.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2E100 /* 107.178.225.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2E180 /* 107.178.225.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2E1A0 /* 107.178.225.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2E1C0 /* 107.178.225.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2E1E0 /* 107.178.225.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2E120 /* 107.178.225.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2E140 /* 107.178.225.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x6BB2E160 /* 107.178.225.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22650600 /* 34.101.6.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22650680 /* 34.101.6.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x226506A0 /* 34.101.6.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22650620 /* 34.101.6.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22650640 /* 34.101.6.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22650660 /* 34.101.6.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741000 /* 34.116.16.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741080 /* 34.116.16.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741020 /* 34.116.16.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741040 /* 34.116.16.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741060 /* 34.116.16.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741100 /* 34.116.17.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741120 /* 34.116.17.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741140 /* 34.116.17.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741200 /* 34.116.18.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741220 /* 34.116.18.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741240 /* 34.116.18.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741300 /* 34.116.19.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741380 /* 34.116.19.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x227413A0 /* 34.116.19.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741320 /* 34.116.19.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741340 /* 34.116.19.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741360 /* 34.116.19.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741400 /* 34.116.20.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741420 /* 34.116.20.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741500 /* 34.116.21.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741520 /* 34.116.21.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741600 /* 34.116.22.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741620 /* 34.116.22.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741640 /* 34.116.22.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741660 /* 34.116.22.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741700 /* 34.116.23.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741720 /* 34.116.23.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741D00 /* 34.116.29.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22741D20 /* 34.116.29.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742000 /* 34.116.32.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742020 /* 34.116.32.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742040 /* 34.116.32.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742100 /* 34.116.33.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742180 /* 34.116.33.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742120 /* 34.116.33.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742140 /* 34.116.33.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742160 /* 34.116.33.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742200 /* 34.116.34.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742220 /* 34.116.34.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742240 /* 34.116.34.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742300 /* 34.116.35.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742320 /* 34.116.35.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742340 /* 34.116.35.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742360 /* 34.116.35.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742900 /* 34.116.41.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742980 /* 34.116.41.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x227429A0 /* 34.116.41.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x227429C0 /* 34.116.41.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x227429E0 /* 34.116.41.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742920 /* 34.116.41.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742940 /* 34.116.41.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742960 /* 34.116.41.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742B00 /* 34.116.43.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742B20 /* 34.116.43.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22742B40 /* 34.116.43.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22400600 /* 34.64.6.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22400680 /* 34.64.6.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x224006A0 /* 34.64.6.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x224006C0 /* 34.64.6.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22400620 /* 34.64.6.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22400640 /* 34.64.6.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22400660 /* 34.64.6.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628800 /* 34.98.136.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628900 /* 34.98.137.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628920 /* 34.98.137.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628940 /* 34.98.137.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628A00 /* 34.98.138.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628A20 /* 34.98.138.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628A40 /* 34.98.138.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628B00 /* 34.98.139.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628B20 /* 34.98.139.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628B40 /* 34.98.139.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628B60 /* 34.98.139.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628C00 /* 34.98.140.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628C80 /* 34.98.140.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628C20 /* 34.98.140.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628C40 /* 34.98.140.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628C60 /* 34.98.140.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628D00 /* 34.98.141.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628D20 /* 34.98.141.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628D40 /* 34.98.141.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628E00 /* 34.98.142.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628E20 /* 34.98.142.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628E40 /* 34.98.142.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628E60 /* 34.98.142.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628F00 /* 34.98.143.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628F80 /* 34.98.143.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628F20 /* 34.98.143.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628F40 /* 34.98.143.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x22628F60 /* 34.98.143.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8000 /* 35.187.128.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8080 /* 35.187.128.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB80A0 /* 35.187.128.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8020 /* 35.187.128.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8040 /* 35.187.128.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8060 /* 35.187.128.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8100 /* 35.187.129.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8180 /* 35.187.129.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB81A0 /* 35.187.129.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB81C0 /* 35.187.129.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB81E0 /* 35.187.129.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8120 /* 35.187.129.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8140 /* 35.187.129.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8160 /* 35.187.129.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8200 /* 35.187.130.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8280 /* 35.187.130.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB82A0 /* 35.187.130.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8220 /* 35.187.130.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8240 /* 35.187.130.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8260 /* 35.187.130.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8300 /* 35.187.131.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8380 /* 35.187.131.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB83A0 /* 35.187.131.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB83C0 /* 35.187.131.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB83E0 /* 35.187.131.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8320 /* 35.187.131.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8340 /* 35.187.131.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8360 /* 35.187.131.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8400 /* 35.187.132.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8480 /* 35.187.132.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB84A0 /* 35.187.132.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB84C0 /* 35.187.132.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB84E0 /* 35.187.132.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8420 /* 35.187.132.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8440 /* 35.187.132.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8460 /* 35.187.132.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8500 /* 35.187.133.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8580 /* 35.187.133.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB85A0 /* 35.187.133.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB85C0 /* 35.187.133.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB85E0 /* 35.187.133.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8520 /* 35.187.133.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8540 /* 35.187.133.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8560 /* 35.187.133.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8600 /* 35.187.134.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8680 /* 35.187.134.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB86A0 /* 35.187.134.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB86C0 /* 35.187.134.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB86E0 /* 35.187.134.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8620 /* 35.187.134.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8640 /* 35.187.134.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8660 /* 35.187.134.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8700 /* 35.187.135.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8780 /* 35.187.135.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8720 /* 35.187.135.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8740 /* 35.187.135.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8760 /* 35.187.135.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8800 /* 35.187.136.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8880 /* 35.187.136.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB88A0 /* 35.187.136.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB88C0 /* 35.187.136.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB88E0 /* 35.187.136.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8900 /* 35.187.137.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8920 /* 35.187.137.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8940 /* 35.187.137.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8960 /* 35.187.137.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8A00 /* 35.187.138.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8A80 /* 35.187.138.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8AA0 /* 35.187.138.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8AC0 /* 35.187.138.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8AE0 /* 35.187.138.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8A20 /* 35.187.138.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8A40 /* 35.187.138.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8A60 /* 35.187.138.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8B00 /* 35.187.139.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8B80 /* 35.187.139.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8BA0 /* 35.187.139.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8BC0 /* 35.187.139.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8BE0 /* 35.187.139.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8B20 /* 35.187.139.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8B40 /* 35.187.139.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8B60 /* 35.187.139.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8C00 /* 35.187.140.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8C80 /* 35.187.140.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8C20 /* 35.187.140.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8C40 /* 35.187.140.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8C60 /* 35.187.140.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8D00 /* 35.187.141.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8D80 /* 35.187.141.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8DA0 /* 35.187.141.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8DC0 /* 35.187.141.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8DE0 /* 35.187.141.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8D20 /* 35.187.141.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8D40 /* 35.187.141.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8D60 /* 35.187.141.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8E00 /* 35.187.142.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8E20 /* 35.187.142.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8E40 /* 35.187.142.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8F00 /* 35.187.143.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8F20 /* 35.187.143.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23BB8F40 /* 35.187.143.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31000 /* 35.243.16.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31020 /* 35.243.16.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31040 /* 35.243.16.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31100 /* 35.243.17.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31120 /* 35.243.17.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31140 /* 35.243.17.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31200 /* 35.243.18.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31220 /* 35.243.18.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31240 /* 35.243.18.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31300 /* 35.243.19.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31320 /* 35.243.19.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31340 /* 35.243.19.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31400 /* 35.243.20.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31480 /* 35.243.20.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F314A0 /* 35.243.20.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31420 /* 35.243.20.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31440 /* 35.243.20.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31460 /* 35.243.20.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31500 /* 35.243.21.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31520 /* 35.243.21.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31600 /* 35.243.22.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31620 /* 35.243.22.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31640 /* 35.243.22.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31700 /* 35.243.23.0/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31780 /* 35.243.23.128/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F317A0 /* 35.243.23.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F317C0 /* 35.243.23.192/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F317E0 /* 35.243.23.224/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31720 /* 35.243.23.32/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31740 /* 35.243.23.64/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x23F31760 /* 35.243.23.96/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ { 0x9D372700 /* 157.55.39.0/24 */, 24, NDPI_HTTP_CRAWLER_BOT },
+ { 0xCF2E0D00 /* 207.46.13.0/24 */, 24, NDPI_HTTP_CRAWLER_BOT },
+ { 0x284DA700 /* 40.77.167.0/24 */, 24, NDPI_HTTP_CRAWLER_BOT },
+ { 0x0D428B00 /* 13.66.139.0/24 */, 24, NDPI_HTTP_CRAWLER_BOT },
+ { 0x0D429000 /* 13.66.144.0/24 */, 24, NDPI_HTTP_CRAWLER_BOT },
+ { 0x34A79000 /* 52.167.144.0/24 */, 24, NDPI_HTTP_CRAWLER_BOT },
+ { 0x0D430A10 /* 13.67.10.16/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x0D4542F0 /* 13.69.66.240/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x0D47ACE0 /* 13.71.172.224/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x8BD93400 /* 139.217.52.0/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0xBFE9CCE0 /* 191.233.204.224/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x14246C20 /* 20.36.108.32/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x142B7810 /* 20.43.120.16/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x284F83D0 /* 40.79.131.208/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x284FBAB0 /* 40.79.186.176/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x34E79400 /* 52.231.148.0/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x144F6BF0 /* 20.79.107.240/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x33694300 /* 51.105.67.0/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x147DA350 /* 20.125.163.80/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x284DBC00 /* 40.77.188.0/22 */, 22, NDPI_HTTP_CRAWLER_BOT },
+ { 0x4137D200 /* 65.55.210.0/24 */, 24, NDPI_HTTP_CRAWLER_BOT },
+ { 0xC71E1800 /* 199.30.24.0/23 */, 23, NDPI_HTTP_CRAWLER_BOT },
+ { 0x284DCA00 /* 40.77.202.0/24 */, 24, NDPI_HTTP_CRAWLER_BOT },
+ { 0x284D8B00 /* 40.77.139.0/25 */, 25, NDPI_HTTP_CRAWLER_BOT },
+ { 0x144AC500 /* 20.74.197.0/28 */, 28, NDPI_HTTP_CRAWLER_BOT },
+ { 0x140F85A0 /* 20.15.133.160/27 */, 27, NDPI_HTTP_CRAWLER_BOT },
+ /* End */
+ { 0x0, 0, 0 }
+};
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index cab17cc5fac..e41e9368b2b 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -73,6 +73,7 @@
 #include "inc_generated/ndpi_ms_skype_teams_match.c.inc"
 #include "inc_generated/ndpi_google_match.c.inc"
 #include "inc_generated/ndpi_google_cloud_match.c.inc"
+#include "inc_generated/ndpi_crawlers_match.c.inc"
 #include "inc_generated/ndpi_icloud_private_relay_match.c.inc"
 #include "inc_generated/ndpi_asn_telegram.c.inc"
 #include "inc_generated/ndpi_asn_apple.c.inc"
@@ -2836,6 +2837,8 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
     if((ndpi_str->ip_risk_ptree = ndpi_patricia_new(32 /* IPv4 */)) != NULL) {
       if(!(prefs & ndpi_dont_load_icloud_private_relay_list)) {
         ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->ip_risk_ptree, ndpi_anonymous_subscriber_protocol_list);
+      if(!(prefs & ndpi_dont_load_crawlers_list))
+        ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->ip_risk_ptree, ndpi_http_crawler_bot_protocol_list);
       }
     }
   }
diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c
index 343c5b0f0ba..499c3df9c09 100644
--- a/src/lib/ndpi_utils.c
+++ b/src/lib/ndpi_utils.c
@@ -2468,6 +2468,26 @@ void ndpi_set_risk(struct ndpi_detection_module_struct *ndpi_str,
 	}
       }
     }
+  } else if(risk_message) {
+    u_int8_t i;
+
+    for(i = 0; i < flow->num_risk_infos; i++)
+      if(flow->risk_infos[i].id == r)
+        return;
+
+    /* Risk already set without any details, but now we have a specific risk_message
+       that we want to save.
+       This might happen with NDPI_HTTP_CRAWLER_BOT which might have been set early via
+       IP matching (no details) and now via UA matching (with message). */
+    if(flow->num_risk_infos < MAX_NUM_RISK_INFOS) {
+      char *s = ndpi_strdup(risk_message);
+
+      if(s != NULL) {
+        flow->risk_infos[flow->num_risk_infos].id = r;
+        flow->risk_infos[flow->num_risk_infos].info = s;
+        flow->num_risk_infos++;
+      }
+    }
   }
 }
 
diff --git a/tests/cfgs/default/result/bot.pcap.out b/tests/cfgs/default/result/bot.pcap.out
index f8a04e636e6..7631073504f 100644
--- a/tests/cfgs/default/result/bot.pcap.out
+++ b/tests/cfgs/default/result/bot.pcap.out
@@ -17,7 +17,7 @@ Automa tls cert:      0/0 (search/found)
 Automa risk mask:     0/0 (search/found)
 Automa common alpns:  0/0 (search/found)
 Patricia risk mask:   2/0 (search/found)
-Patricia risk:        2/0 (search/found)
+Patricia risk:        1/1 (search/found)
 Patricia protocols:   2/1 (search/found)
 
 HTTP	402	431124	1
diff --git a/tests/cfgs/default/result/ssh.pcap.out b/tests/cfgs/default/result/ssh.pcap.out
index 7c30400d31a..22ff4cc04db 100644
--- a/tests/cfgs/default/result/ssh.pcap.out
+++ b/tests/cfgs/default/result/ssh.pcap.out
@@ -22,4 +22,4 @@ Patricia protocols:   2/0 (search/found)
 
 SSH	258	35546	1
 
-	1	TCP 172.16.238.1:58395 <-> 172.16.238.168:22 [proto: 92/SSH][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 10][cat: RemoteAccess/12][159 pkts/15615 bytes <-> 99 pkts/19931 bytes][Goodput ratio: 33/67][248.48 sec][Hostname/SNI: SSH-2.0-OpenSSH_5.3][bytes ratio: -0.121 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 1846/2934 166223/166224 14794/19692][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 98/201 970/1346 83/283][Risk: ** SSH Obsolete Cli Vers/Cipher **** SSH Obsolete Ser Vers/Cipher **][Risk Score: 150][HASSH-C: 21B457A327CE7A2D4FCE5EF2C42400BD][Server: SSH-2.0-OpenSSH_5.6][HASSH-S: B1C6C0D56317555B85C7005A3DE29325][Plen Bins: 2,76,12,2,3,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0]
+	1	TCP 172.16.238.1:58395 <-> 172.16.238.168:22 [proto: 92/SSH][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 10][cat: RemoteAccess/12][159 pkts/15615 bytes <-> 99 pkts/19931 bytes][Goodput ratio: 33/67][248.48 sec][Hostname/SNI: SSH-2.0-OpenSSH_5.3][bytes ratio: -0.121 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 1846/2934 166223/166224 14794/19692][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 98/201 970/1346 83/283][Risk: ** SSH Obsolete Cli Vers/Cipher **** SSH Obsolete Ser Vers/Cipher **][Risk Score: 150][Risk Info: Found cipher arcfour128 / Found cipher arcfour128][HASSH-C: 21B457A327CE7A2D4FCE5EF2C42400BD][Server: SSH-2.0-OpenSSH_5.6][HASSH-S: B1C6C0D56317555B85C7005A3DE29325][Plen Bins: 2,76,12,2,3,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0]
diff --git a/utils/crawlers_ip_addresses_download.sh b/utils/crawlers_ip_addresses_download.sh
new file mode 100755
index 00000000000..8087eaa8fce
--- /dev/null
+++ b/utils/crawlers_ip_addresses_download.sh
@@ -0,0 +1,60 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "${0}")" || exit 1
+
+DEST=../src/lib/inc_generated/ndpi_crawlers_match.c.inc
+TMP1=/tmp/bot_google_c1.json
+TMP2=/tmp/bot_google_c2.json
+TMP3=/tmp/bot_google_c3.json
+TMP_BING=/tmp/bot_bing.json
+LIST=/tmp/bot.list
+#Google Common crawlers
+ORIGIN1="https://developers.google.com/static/search/apis/ipranges/googlebot.json"
+#Google Special-case crawlers
+ORIGIN2="https://developers.google.com/static/search/apis/ipranges/special-crawlers.json"
+#Google User-triggered fetchers
+ORIGIN3="https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json"
+#Bing Bot
+ORIGIN_BING="https://www.bing.com/toolbox/bingbot.json"
+
+
+echo "(1) Downloading file... ${ORIGIN1}"
+http_response=$(curl -s -o $TMP1 -w "%{http_code}" ${ORIGIN1})
+if [ "$http_response" != "200" ]; then
+    echo "Error $http_response: you probably need to update the list url!"
+    exit 1
+fi
+
+echo "(1) Downloading file... ${ORIGIN2}"
+http_response=$(curl -s -o $TMP2 -w "%{http_code}" ${ORIGIN2})
+if [ "$http_response" != "200" ]; then
+    echo "Error $http_response: you probably need to update the list url!"
+    exit 1
+fi
+
+echo "(1) Downloading file... ${ORIGIN3}"
+http_response=$(curl -s -o $TMP3 -w "%{http_code}" ${ORIGIN3})
+if [ "$http_response" != "200" ]; then
+    echo "Error $http_response: you probably need to update the list url!"
+    exit 1
+fi
+
+echo "(1) Downloading file... ${ORIGIN_BING}"
+http_response=$(curl -s -o $TMP_BING -w "%{http_code}" ${ORIGIN_BING})
+if [ "$http_response" != "200" ]; then
+    echo "Error $http_response: you probably need to update the list url!"
+    exit 1
+fi
+
+echo "(2) Processing IP addresses..."
+jq -r '.prefixes | .[].ipv4Prefix  | select( . != null )' $TMP1 > $LIST # TODO: ipv6
+jq -r '.prefixes | .[].ipv4Prefix  | select( . != null )' $TMP2 >> $LIST # TODO: ipv6
+jq -r '.prefixes | .[].ipv4Prefix  | select( . != null )' $TMP3 >> $LIST # TODO: ipv6
+jq -r '.prefixes | .[].ipv4Prefix  | select( . != null )' $TMP_BING >> $LIST # TODO: ipv6
+./ipaddr2list.py $LIST NDPI_HTTP_CRAWLER_BOT > $DEST
+rm -f $TMP1 $TMP2 $TMP3 $TMP_BING $LIST
+
+echo "(3) Crawlers IPs are available in $DEST"
+exit 0
diff --git a/utils/update_every_lists.sh b/utils/update_every_lists.sh
index 514a2999a86..e9702f1b191 100755
--- a/utils/update_every_lists.sh
+++ b/utils/update_every_lists.sh
@@ -28,6 +28,8 @@ RETVAL=$(( RETVAL + $? ))
 RETVAL=$(( RETVAL + $? ))
 ./icloud_private_relay_ip_addresses_download.sh
 RETVAL=$(( RETVAL + $? ))
+./crawlers_ip_addresses_download.sh
+RETVAL=$(( RETVAL + $? ))
 
 ./asn_update.sh
 RETVAL=$(( RETVAL + $? ))