From 887fa78a44cfdd18d31605522d76196d08aea5f1 Mon Sep 17 00:00:00 2001 From: Nardi Ivan Date: Sat, 19 Feb 2022 11:42:48 +0100 Subject: [PATCH] Add some scripts to easily update some IPs lists While the lists in a6ff0dd0 and 2f5f445f are somehow provided by the companies themselves (or by some interested parties), these new lists are directly extracted from BGP information, via AS prefixes. *Usually*, these new lists are far more stable than the previous ones. TODO: * add some other ASNs (see `src/lib/ndpi_content_match.c.inc`) * IPv6, as usual :-( --- src/include/ndpi_typedefs.h | 1 + src/lib/ndpi_asn_apple.c.inc | 36 +++++ src/lib/ndpi_asn_facebook.c.inc | 75 +++++++++++ src/lib/ndpi_asn_netflix.c.inc | 40 ++++++ src/lib/ndpi_asn_teamviewer.c.inc | 33 +++++ src/lib/ndpi_asn_telegram.c.inc | 34 +++++ src/lib/ndpi_asn_twitter.c.inc | 60 +++++++++ src/lib/ndpi_asn_webex.c.inc | 51 +++++++ src/lib/ndpi_content_match.c.inc | 124 +----------------- src/lib/ndpi_main.c | 16 +++ utils/asn_apple.sh | 20 +++ utils/asn_facebook.sh | 21 +++ utils/asn_netflix.sh | 21 +++ utils/asn_teamviewer.sh | 21 +++ utils/asn_telegram.sh | 22 ++++ utils/asn_twitter.sh | 21 +++ utils/asn_webex.sh | 21 +++ utils/get_routes_by_asn.sh | 19 +++ utils/mergeipaddrlist.py | 19 +++ ...t_match_lists.sh => update_every_lists.sh} | 10 +- 20 files changed, 545 insertions(+), 120 deletions(-) create mode 100644 src/lib/ndpi_asn_apple.c.inc create mode 100644 src/lib/ndpi_asn_facebook.c.inc create mode 100644 src/lib/ndpi_asn_netflix.c.inc create mode 100644 src/lib/ndpi_asn_teamviewer.c.inc create mode 100644 src/lib/ndpi_asn_telegram.c.inc create mode 100644 src/lib/ndpi_asn_twitter.c.inc create mode 100644 src/lib/ndpi_asn_webex.c.inc create mode 100755 utils/asn_apple.sh create mode 100755 utils/asn_facebook.sh create mode 100755 utils/asn_netflix.sh create mode 100755 utils/asn_teamviewer.sh create mode 100755 utils/asn_telegram.sh create mode 100755 utils/asn_twitter.sh create mode 100755 utils/asn_webex.sh create mode 100755 utils/get_routes_by_asn.sh create mode 100755 utils/mergeipaddrlist.py rename utils/{update_every_content_match_lists.sh => update_every_lists.sh} (69%) diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index e19e9f82cb0..bbd26961c57 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -1402,6 +1402,7 @@ typedef enum ndpi_dont_load_microsoft_list = (1 << 9), ndpi_dont_load_google_list = (1 << 10), ndpi_dont_load_google_cloud_list = (1 << 11), + ndpi_dont_load_asn_lists = (1 << 12), } ndpi_prefs; typedef struct { diff --git a/src/lib/ndpi_asn_apple.c.inc b/src/lib/ndpi_asn_apple.c.inc new file mode 100644 index 00000000000..80adb10eb31 --- /dev/null +++ b/src/lib/ndpi_asn_apple.c.inc @@ -0,0 +1,36 @@ +/* + * + * This file is generated automatically and part of nDPI + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see . + * + */ + +/* ****************************************************** */ + + +static ndpi_network ndpi_protocol_apple_protocol_list[] = { + { 0x11000000 /* 17.0.0.0/8 */, 8, NDPI_PROTOCOL_APPLE }, + { 0x39660000 /* 57.102.0.0/15 */, 15, NDPI_PROTOCOL_APPLE }, + { 0x39700000 /* 57.112.0.0/12 */, 12, NDPI_PROTOCOL_APPLE }, + { 0x3F5CE000 /* 63.92.224.0/19 */, 19, NDPI_PROTOCOL_APPLE }, + { 0x41C71600 /* 65.199.22.0/23 */, 23, NDPI_PROTOCOL_APPLE }, + { 0x8BB28000 /* 139.178.128.0/18 */, 18, NDPI_PROTOCOL_APPLE }, + { 0x90B20000 /* 144.178.0.0/19 */, 19, NDPI_PROTOCOL_APPLE }, + { 0x90B22000 /* 144.178.32.0/24 */, 24, NDPI_PROTOCOL_APPLE }, + { 0x90B22400 /* 144.178.36.0/22 */, 22, NDPI_PROTOCOL_APPLE }, + { 0x90B23000 /* 144.178.48.0/20 */, 20, NDPI_PROTOCOL_APPLE }, + /* End */ + { 0x0, 0, 0 } +}; diff --git a/src/lib/ndpi_asn_facebook.c.inc b/src/lib/ndpi_asn_facebook.c.inc new file mode 100644 index 00000000000..563a0f56c19 --- /dev/null +++ b/src/lib/ndpi_asn_facebook.c.inc @@ -0,0 +1,75 @@ +/* + * + * This file is generated automatically and part of nDPI + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see . + * + */ + +/* ****************************************************** */ + + +static ndpi_network ndpi_protocol_facebook_protocol_list[] = { + { 0x1F0D1800 /* 31.13.24.0/21 */, 21, NDPI_PROTOCOL_FACEBOOK }, + { 0x1F0D4000 /* 31.13.64.0/18 */, 18, NDPI_PROTOCOL_FACEBOOK }, + { 0x2D402800 /* 45.64.40.0/22 */, 22, NDPI_PROTOCOL_FACEBOOK }, + { 0x42DC9000 /* 66.220.144.0/20 */, 20, NDPI_PROTOCOL_FACEBOOK }, + { 0x453FB000 /* 69.63.176.0/20 */, 20, NDPI_PROTOCOL_FACEBOOK }, + { 0x45ABE000 /* 69.171.224.0/19 */, 19, NDPI_PROTOCOL_FACEBOOK }, + { 0x4A774C00 /* 74.119.76.0/22 */, 22, NDPI_PROTOCOL_FACEBOOK }, + { 0x66846000 /* 102.132.96.0/20 */, 20, NDPI_PROTOCOL_FACEBOOK }, + { 0x66847000 /* 102.132.112.0/21 */, 21, NDPI_PROTOCOL_FACEBOOK }, + { 0x66847800 /* 102.132.120.0/24 */, 24, NDPI_PROTOCOL_FACEBOOK }, + { 0x66847A00 /* 102.132.122.0/24 */, 24, NDPI_PROTOCOL_FACEBOOK }, + { 0x66847C00 /* 102.132.124.0/24 */, 24, NDPI_PROTOCOL_FACEBOOK }, + { 0x66847E00 /* 102.132.126.0/23 */, 23, NDPI_PROTOCOL_FACEBOOK }, + { 0x66DDBC00 /* 102.221.188.0/23 */, 23, NDPI_PROTOCOL_FACEBOOK }, + { 0x66DDBF00 /* 102.221.191.0/24 */, 24, NDPI_PROTOCOL_FACEBOOK }, + { 0x67046000 /* 103.4.96.0/22 */, 22, NDPI_PROTOCOL_FACEBOOK }, + { 0x81860000 /* 129.134.0.0/17 */, 17, NDPI_PROTOCOL_FACEBOOK }, + { 0x81868000 /* 129.134.128.0/24 */, 24, NDPI_PROTOCOL_FACEBOOK }, + { 0x81868200 /* 129.134.130.0/23 */, 23, NDPI_PROTOCOL_FACEBOOK }, + { 0x81868400 /* 129.134.132.0/24 */, 24, NDPI_PROTOCOL_FACEBOOK }, + { 0x81868700 /* 129.134.135.0/24 */, 24, NDPI_PROTOCOL_FACEBOOK }, + { 0x81868800 /* 129.134.136.0/22 */, 22, NDPI_PROTOCOL_FACEBOOK }, + { 0x81868C00 /* 129.134.140.0/24 */, 24, NDPI_PROTOCOL_FACEBOOK }, + { 0x81868F00 /* 129.134.143.0/24 */, 24, NDPI_PROTOCOL_FACEBOOK }, + { 0x81869000 /* 129.134.144.0/24 */, 24, NDPI_PROTOCOL_FACEBOOK }, + { 0x81869300 /* 129.134.147.0/24 */, 24, NDPI_PROTOCOL_FACEBOOK }, + { 0x81869400 /* 129.134.148.0/23 */, 23, NDPI_PROTOCOL_FACEBOOK }, + { 0x81869600 /* 129.134.150.0/24 */, 24, NDPI_PROTOCOL_FACEBOOK }, + { 0x81869A00 /* 129.134.154.0/23 */, 23, NDPI_PROTOCOL_FACEBOOK }, + { 0x81869C00 /* 129.134.156.0/22 */, 22, NDPI_PROTOCOL_FACEBOOK }, + { 0x8186A000 /* 129.134.160.0/24 */, 24, NDPI_PROTOCOL_FACEBOOK }, + { 0x8186A300 /* 129.134.163.0/24 */, 24, NDPI_PROTOCOL_FACEBOOK }, + { 0x8186A400 /* 129.134.164.0/23 */, 23, NDPI_PROTOCOL_FACEBOOK }, + { 0x8186AB00 /* 129.134.171.0/24 */, 24, NDPI_PROTOCOL_FACEBOOK }, + { 0x9DF00000 /* 157.240.0.0/17 */, 17, NDPI_PROTOCOL_FACEBOOK }, + { 0x9DF08000 /* 157.240.128.0/24 */, 24, NDPI_PROTOCOL_FACEBOOK }, + { 0x9DF09E00 /* 157.240.158.0/23 */, 23, NDPI_PROTOCOL_FACEBOOK }, + { 0x9DF0AC00 /* 157.240.172.0/24 */, 24, NDPI_PROTOCOL_FACEBOOK }, + { 0x9DF0AE00 /* 157.240.174.0/23 */, 23, NDPI_PROTOCOL_FACEBOOK }, + { 0x9DF0B000 /* 157.240.176.0/22 */, 22, NDPI_PROTOCOL_FACEBOOK }, + { 0x9DF0B400 /* 157.240.180.0/23 */, 23, NDPI_PROTOCOL_FACEBOOK }, + { 0x9DF0C000 /* 157.240.192.0/18 */, 18, NDPI_PROTOCOL_FACEBOOK }, + { 0xA3728000 /* 163.114.128.0/20 */, 20, NDPI_PROTOCOL_FACEBOOK }, + { 0xADFC4000 /* 173.252.64.0/18 */, 18, NDPI_PROTOCOL_FACEBOOK }, + { 0xB33CC000 /* 179.60.192.0/22 */, 22, NDPI_PROTOCOL_FACEBOOK }, + { 0xB93CD800 /* 185.60.216.0/22 */, 22, NDPI_PROTOCOL_FACEBOOK }, + { 0xB959DA00 /* 185.89.218.0/23 */, 23, NDPI_PROTOCOL_FACEBOOK }, + { 0xC7C94000 /* 199.201.64.0/22 */, 22, NDPI_PROTOCOL_FACEBOOK }, + { 0xCC0F1400 /* 204.15.20.0/22 */, 22, NDPI_PROTOCOL_FACEBOOK }, + /* End */ + { 0x0, 0, 0 } +}; diff --git a/src/lib/ndpi_asn_netflix.c.inc b/src/lib/ndpi_asn_netflix.c.inc new file mode 100644 index 00000000000..d68b994f455 --- /dev/null +++ b/src/lib/ndpi_asn_netflix.c.inc @@ -0,0 +1,40 @@ +/* + * + * This file is generated automatically and part of nDPI + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see . + * + */ + +/* ****************************************************** */ + + +static ndpi_network ndpi_protocol_netflix_protocol_list[] = { + { 0x17F60000 /* 23.246.0.0/18 */, 18, NDPI_PROTOCOL_NETFLIX }, + { 0x254DB800 /* 37.77.184.0/21 */, 21, NDPI_PROTOCOL_NETFLIX }, + { 0x2D390000 /* 45.57.0.0/17 */, 17, NDPI_PROTOCOL_NETFLIX }, + { 0x40788000 /* 64.120.128.0/17 */, 17, NDPI_PROTOCOL_NETFLIX }, + { 0x42C58000 /* 66.197.128.0/17 */, 17, NDPI_PROTOCOL_NETFLIX }, + { 0x4535E000 /* 69.53.224.0/19 */, 19, NDPI_PROTOCOL_NETFLIX }, + { 0x6CAF2000 /* 108.175.32.0/20 */, 20, NDPI_PROTOCOL_NETFLIX }, + { 0xB902DC00 /* 185.2.220.0/22 */, 22, NDPI_PROTOCOL_NETFLIX }, + { 0xB909BC00 /* 185.9.188.0/22 */, 22, NDPI_PROTOCOL_NETFLIX }, + { 0xC0AD4000 /* 192.173.64.0/18 */, 18, NDPI_PROTOCOL_NETFLIX }, + { 0xC6266000 /* 198.38.96.0/19 */, 19, NDPI_PROTOCOL_NETFLIX }, + { 0xC62D3000 /* 198.45.48.0/20 */, 20, NDPI_PROTOCOL_NETFLIX }, + { 0xCF2D4800 /* 207.45.72.0/22 */, 22, NDPI_PROTOCOL_NETFLIX }, + { 0xD04B4C00 /* 208.75.76.0/22 */, 22, NDPI_PROTOCOL_NETFLIX }, + /* End */ + { 0x0, 0, 0 } +}; diff --git a/src/lib/ndpi_asn_teamviewer.c.inc b/src/lib/ndpi_asn_teamviewer.c.inc new file mode 100644 index 00000000000..6306194137a --- /dev/null +++ b/src/lib/ndpi_asn_teamviewer.c.inc @@ -0,0 +1,33 @@ +/* + * + * This file is generated automatically and part of nDPI + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see . + * + */ + +/* ****************************************************** */ + + +static ndpi_network ndpi_protocol_teamviewer_protocol_list[] = { + { 0x2D936C00 /* 45.147.108.0/22 */, 22, NDPI_PROTOCOL_TEAMVIEWER }, + { 0xB9BC2000 /* 185.188.32.0/22 */, 22, NDPI_PROTOCOL_TEAMVIEWER }, + { 0xB9F51C00 /* 185.245.28.0/23 */, 23, NDPI_PROTOCOL_TEAMVIEWER }, + { 0xB9F51E00 /* 185.245.30.0/24 */, 24, NDPI_PROTOCOL_TEAMVIEWER }, + { 0xC120E100 /* 193.32.225.0/24 */, 24, NDPI_PROTOCOL_TEAMVIEWER }, + { 0xC120E200 /* 193.32.226.0/23 */, 23, NDPI_PROTOCOL_TEAMVIEWER }, + { 0xC13A6400 /* 193.58.100.0/23 */, 23, NDPI_PROTOCOL_TEAMVIEWER }, + /* End */ + { 0x0, 0, 0 } +}; diff --git a/src/lib/ndpi_asn_telegram.c.inc b/src/lib/ndpi_asn_telegram.c.inc new file mode 100644 index 00000000000..ed375805e89 --- /dev/null +++ b/src/lib/ndpi_asn_telegram.c.inc @@ -0,0 +1,34 @@ +/* + * + * This file is generated automatically and part of nDPI + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see . + * + */ + +/* ****************************************************** */ + + +static ndpi_network ndpi_protocol_telegram_protocol_list[] = { + { 0x5B69C000 /* 91.105.192.0/23 */, 23, NDPI_PROTOCOL_TELEGRAM }, + { 0x5B6C0400 /* 91.108.4.0/22 */, 22, NDPI_PROTOCOL_TELEGRAM }, + { 0x5B6C0800 /* 91.108.8.0/21 */, 21, NDPI_PROTOCOL_TELEGRAM }, + { 0x5B6C1000 /* 91.108.16.0/21 */, 21, NDPI_PROTOCOL_TELEGRAM }, + { 0x5B6C3800 /* 91.108.56.0/22 */, 22, NDPI_PROTOCOL_TELEGRAM }, + { 0x5FA14000 /* 95.161.64.0/20 */, 20, NDPI_PROTOCOL_TELEGRAM }, + { 0x959AA000 /* 149.154.160.0/20 */, 20, NDPI_PROTOCOL_TELEGRAM }, + { 0xB94C9700 /* 185.76.151.0/24 */, 24, NDPI_PROTOCOL_TELEGRAM }, + /* End */ + { 0x0, 0, 0 } +}; diff --git a/src/lib/ndpi_asn_twitter.c.inc b/src/lib/ndpi_asn_twitter.c.inc new file mode 100644 index 00000000000..c0b5c6222da --- /dev/null +++ b/src/lib/ndpi_asn_twitter.c.inc @@ -0,0 +1,60 @@ +/* + * + * This file is generated automatically and part of nDPI + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see . + * + */ + +/* ****************************************************** */ + + +static ndpi_network ndpi_protocol_twitter_protocol_list[] = { + { 0x0819C200 /* 8.25.194.0/23 */, 23, NDPI_PROTOCOL_TWITTER }, + { 0x0819C400 /* 8.25.196.0/23 */, 23, NDPI_PROTOCOL_TWITTER }, + { 0x403F0000 /* 64.63.0.0/18 */, 18, NDPI_PROTOCOL_TWITTER }, + { 0x450C3800 /* 69.12.56.0/21 */, 21, NDPI_PROTOCOL_TWITTER }, + { 0x45C3A000 /* 69.195.160.0/24 */, 24, NDPI_PROTOCOL_TWITTER }, + { 0x45C3A200 /* 69.195.162.0/23 */, 23, NDPI_PROTOCOL_TWITTER }, + { 0x45C3A400 /* 69.195.164.0/23 */, 23, NDPI_PROTOCOL_TWITTER }, + { 0x45C3A600 /* 69.195.166.0/24 */, 24, NDPI_PROTOCOL_TWITTER }, + { 0x45C3A800 /* 69.195.168.0/23 */, 23, NDPI_PROTOCOL_TWITTER }, + { 0x45C3AB00 /* 69.195.171.0/24 */, 24, NDPI_PROTOCOL_TWITTER }, + { 0x45C3AC00 /* 69.195.172.0/24 */, 24, NDPI_PROTOCOL_TWITTER }, + { 0x45C3AE00 /* 69.195.174.0/23 */, 23, NDPI_PROTOCOL_TWITTER }, + { 0x45C3B000 /* 69.195.176.0/20 */, 20, NDPI_PROTOCOL_TWITTER }, + { 0x67FC7000 /* 103.252.112.0/22 */, 22, NDPI_PROTOCOL_TWITTER }, + { 0x68F42800 /* 104.244.40.0/23 */, 23, NDPI_PROTOCOL_TWITTER }, + { 0x68F42A00 /* 104.244.42.0/24 */, 24, NDPI_PROTOCOL_TWITTER }, + { 0x68F42C00 /* 104.244.44.0/22 */, 22, NDPI_PROTOCOL_TWITTER }, + { 0xB92D0400 /* 185.45.4.0/22 */, 22, NDPI_PROTOCOL_TWITTER }, + { 0xBC40E000 /* 188.64.224.0/21 */, 21, NDPI_PROTOCOL_TWITTER }, + { 0xC02C4500 /* 192.44.69.0/24 */, 24, NDPI_PROTOCOL_TWITTER }, + { 0xC030EC00 /* 192.48.236.0/23 */, 23, NDPI_PROTOCOL_TWITTER }, + { 0xC0854C00 /* 192.133.76.0/22 */, 22, NDPI_PROTOCOL_TWITTER }, + { 0xC7109C00 /* 199.16.156.0/22 */, 22, NDPI_PROTOCOL_TWITTER }, + { 0xC73B9400 /* 199.59.148.0/22 */, 22, NDPI_PROTOCOL_TWITTER }, + { 0xC7603800 /* 199.96.56.0/21 */, 21, NDPI_PROTOCOL_TWITTER }, + { 0xCAA08000 /* 202.160.128.0/22 */, 22, NDPI_PROTOCOL_TWITTER }, + { 0xD1EDC000 /* 209.237.192.0/21 */, 21, NDPI_PROTOCOL_TWITTER }, + { 0xD1EDC800 /* 209.237.200.0/22 */, 22, NDPI_PROTOCOL_TWITTER }, + { 0xD1EDCC00 /* 209.237.204.0/23 */, 23, NDPI_PROTOCOL_TWITTER }, + { 0xD1EDCE00 /* 209.237.206.0/24 */, 24, NDPI_PROTOCOL_TWITTER }, + { 0xD1EDD000 /* 209.237.208.0/22 */, 22, NDPI_PROTOCOL_TWITTER }, + { 0xD1EDD400 /* 209.237.212.0/24 */, 24, NDPI_PROTOCOL_TWITTER }, + { 0xD1EDD600 /* 209.237.214.0/23 */, 23, NDPI_PROTOCOL_TWITTER }, + { 0xD1EDD800 /* 209.237.216.0/21 */, 21, NDPI_PROTOCOL_TWITTER }, + /* End */ + { 0x0, 0, 0 } +}; diff --git a/src/lib/ndpi_asn_webex.c.inc b/src/lib/ndpi_asn_webex.c.inc new file mode 100644 index 00000000000..0f947c011aa --- /dev/null +++ b/src/lib/ndpi_asn_webex.c.inc @@ -0,0 +1,51 @@ +/* + * + * This file is generated automatically and part of nDPI + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see . + * + */ + +/* ****************************************************** */ + + +static ndpi_network ndpi_protocol_webex_protocol_list[] = { + { 0x17590000 /* 23.89.0.0/17 */, 17, NDPI_PROTOCOL_WEBEX }, + { 0x17598000 /* 23.89.128.0/19 */, 19, NDPI_PROTOCOL_WEBEX }, + { 0x1759A000 /* 23.89.160.0/20 */, 20, NDPI_PROTOCOL_WEBEX }, + { 0x1759B000 /* 23.89.176.0/21 */, 21, NDPI_PROTOCOL_WEBEX }, + { 0x1759C000 /* 23.89.192.0/19 */, 19, NDPI_PROTOCOL_WEBEX }, + { 0x3E6DC000 /* 62.109.192.0/18 */, 18, NDPI_PROTOCOL_WEBEX }, + { 0x40446000 /* 64.68.96.0/19 */, 19, NDPI_PROTOCOL_WEBEX }, + { 0x4272A000 /* 66.114.160.0/20 */, 20, NDPI_PROTOCOL_WEBEX }, + { 0x42A32000 /* 66.163.32.0/19 */, 19, NDPI_PROTOCOL_WEBEX }, + { 0x451AA000 /* 69.26.160.0/19 */, 19, NDPI_PROTOCOL_WEBEX }, + { 0x721DC000 /* 114.29.192.0/19 */, 19, NDPI_PROTOCOL_WEBEX }, + { 0x8BB15C00 /* 139.177.92.0/23 */, 23, NDPI_PROTOCOL_WEBEX }, + { 0x96FD8000 /* 150.253.128.0/17 */, 17, NDPI_PROTOCOL_WEBEX }, + { 0xAA480000 /* 170.72.0.0/17 */, 17, NDPI_PROTOCOL_WEBEX }, + { 0xAA488000 /* 170.72.128.0/18 */, 18, NDPI_PROTOCOL_WEBEX }, + { 0xAA48C000 /* 170.72.192.0/19 */, 19, NDPI_PROTOCOL_WEBEX }, + { 0xAA858000 /* 170.133.128.0/18 */, 18, NDPI_PROTOCOL_WEBEX }, + { 0xAD27E000 /* 173.39.224.0/19 */, 19, NDPI_PROTOCOL_WEBEX }, + { 0xADF30000 /* 173.243.0.0/20 */, 20, NDPI_PROTOCOL_WEBEX }, + { 0xCFB6A000 /* 207.182.160.0/19 */, 19, NDPI_PROTOCOL_WEBEX }, + { 0xD0FDFE00 /* 208.253.254.0/24 */, 24, NDPI_PROTOCOL_WEBEX }, + { 0xD0FE7200 /* 208.254.114.0/24 */, 24, NDPI_PROTOCOL_WEBEX }, + { 0xD1C5C000 /* 209.197.192.0/19 */, 19, NDPI_PROTOCOL_WEBEX }, + { 0xD204C000 /* 210.4.192.0/20 */, 20, NDPI_PROTOCOL_WEBEX }, + { 0xD8978000 /* 216.151.128.0/19 */, 19, NDPI_PROTOCOL_WEBEX }, + /* End */ + { 0x0, 0, 0 } +}; diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc index 06c45ba5ae7..323d7116e63 100644 --- a/src/lib/ndpi_content_match.c.inc +++ b/src/lib/ndpi_content_match.c.inc @@ -28,31 +28,10 @@ static ndpi_network host_protocol_list[] = { { 0x23A06456 /* 35.160.100.86 */, 32, NDPI_PROTOCOL_SOUNDCLOUD }, { 0x36C0CA58 /* 54.192.202.88 */, 32, NDPI_PROTOCOL_SOUNDCLOUD }, - /* - Facebook, Inc. - origin AS32934, AS54115 - */ - - { 0x1F0D1800 /* 31.13.24.0/21 */, 21, NDPI_PROTOCOL_FACEBOOK }, - { 0x1F0D4000 /* 31.13.64.0/18 */, 18, NDPI_PROTOCOL_FACEBOOK }, - { 0x2D402800 /* 45.64.40.0/22 */, 22, NDPI_PROTOCOL_FACEBOOK }, - { 0x42DC9000 /* 66.220.144.0/20 */, 20, NDPI_PROTOCOL_FACEBOOK }, - { 0x453FB000 /* 69.63.176.0/20 */, 20, NDPI_PROTOCOL_FACEBOOK }, - { 0x45ABE000 /* 69.171.224.0/19 */, 19, NDPI_PROTOCOL_FACEBOOK }, - { 0x4A774C00 /* 74.119.76.0/22 */, 22, NDPI_PROTOCOL_FACEBOOK }, - { 0x67046000 /* 103.4.96.0/22 */, 22, NDPI_PROTOCOL_FACEBOOK }, - { 0x81860000 /* 129.134.0.0/16 */, 16, NDPI_PROTOCOL_FACEBOOK }, - { 0x81CD5E00 /* 129.205.94.0/23 */, 23, NDPI_PROTOCOL_FACEBOOK }, - { 0x9df00000 /* 157.240.0.0/16 */, 16, NDPI_PROTOCOL_FACEBOOK }, - { 0xADFC4000 /* 173.252.64.0/18 */, 18, NDPI_PROTOCOL_FACEBOOK }, - { 0xB33CC000 /* 179.60.192.0/22 */, 22, NDPI_PROTOCOL_FACEBOOK }, - { 0xB93CD800 /* 185.60.216.0/22 */, 22, NDPI_PROTOCOL_FACEBOOK }, - { 0xC7C94000 /* 199.201.64.0/22 */, 22, NDPI_PROTOCOL_FACEBOOK }, - { 0xCC0F1400 /* 204.15.20.0/22 */, 22, NDPI_PROTOCOL_FACEBOOK }, - /* Instagram, Inc. - origin AS32934 (instagram-*.fbcdn.net, fna-instagram-*.fbcdn.net) + (instagram-*.fbcdn.net, fna-instagram-*.fbcdn.net) + TODO: are we able to autogenerate this list (like the other FB IPs)? */ { 0x9DF00225 /* 157.240.2.37/32 */, 32, NDPI_PROTOCOL_INSTAGRAM }, { 0x9DF00234 /* 157.240.2.52/32 */, 32, NDPI_PROTOCOL_INSTAGRAM }, @@ -513,35 +492,6 @@ static ndpi_network host_protocol_list[] = { { 0x9DF0F1AE /* 157.240.241.174/32 */, 32, NDPI_PROTOCOL_INSTAGRAM }, { 0x9DF0F1C4 /* 157.240.241.196/32 */, 32, NDPI_PROTOCOL_INSTAGRAM }, - - /* - Twitter Inc. - origin AS13414, AS35995 - */ - - { 0x0819C200 /* 8.25.194.0/23 */, 23, NDPI_PROTOCOL_TWITTER }, - { 0x0819C400 /* 8.25.196.0/23 */, 23, NDPI_PROTOCOL_TWITTER }, - { 0x450C3800 /* 69.12.56.0/21 */, 21, NDPI_PROTOCOL_TWITTER }, - { 0x67FC7000 /* 103.252.112.0/22 */, 22, NDPI_PROTOCOL_TWITTER }, - { 0x68F42800 /* 104.244.40.0/24 */, 24, NDPI_PROTOCOL_TWITTER }, - { 0x68F42900 /* 104.244.41.0/24 */, 24, NDPI_PROTOCOL_TWITTER }, - { 0x68F42A00 /* 104.244.42.0/24 */, 24, NDPI_PROTOCOL_TWITTER }, - { 0x68F42B00 /* 104.244.43.0/24 */, 24, NDPI_PROTOCOL_TWITTER }, - { 0x68F42C00 /* 104.244.44.0/24 */, 24, NDPI_PROTOCOL_TWITTER }, - { 0x68F42D00 /* 104.244.45.0/24 */, 24, NDPI_PROTOCOL_TWITTER }, - { 0x68F42E00 /* 104.244.46.0/24 */, 24, NDPI_PROTOCOL_TWITTER }, - { 0x68F42F00 /* 104.244.47.0/24 */, 24, NDPI_PROTOCOL_TWITTER }, - { 0xB92D0400 /* 185.45.4.0/23 */, 23, NDPI_PROTOCOL_TWITTER }, - { 0xB92D0600 /* 185.45.6.0/23 */, 23, NDPI_PROTOCOL_TWITTER }, - { 0xBC40E000 /* 188.64.224.0/21 */, 21, NDPI_PROTOCOL_TWITTER }, - { 0xC02C4400 /* 192.44.68.0/23 */, 23, NDPI_PROTOCOL_TWITTER }, - { 0xC0854C00 /* 192.133.76.0/22 */, 22, NDPI_PROTOCOL_TWITTER }, - { 0xC7109C00 /* 199.16.156.0/22 */, 22, NDPI_PROTOCOL_TWITTER }, - { 0xC73B9400 /* 199.59.148.0/22 */, 22, NDPI_PROTOCOL_TWITTER }, - { 0xC7453A00 /* 199.69.58.0/23 */, 23, NDPI_PROTOCOL_TWITTER }, - { 0xC7603800 /* 199.96.56.0/21 */, 21, NDPI_PROTOCOL_TWITTER }, - { 0xCAA08000 /* 202.160.128.0/22 */, 22, NDPI_PROTOCOL_TWITTER }, - /* WeChat origin AS132203, AS132591, AS45090 @@ -570,26 +520,6 @@ static ndpi_network host_protocol_list[] = { { 0xD043D800 /* 208.67.216.0/21 */, 21, NDPI_PROTOCOL_OPENDNS }, { 0xD0452000 /* 208.69.32.0/21 */, 21, NDPI_PROTOCOL_OPENDNS }, - /* - Netflix Inc. / Netflix Streaming Services Inc. - origin AS2906, AS55095, AS40027 - */ - - { 0x17F60000 /* 23.246.0.0/18 */, 18, NDPI_PROTOCOL_NETFLIX }, - { 0x254DB800 /* 37.77.184.0/21 */, 21, NDPI_PROTOCOL_NETFLIX }, - { 0x26487E00 /* 38.72.126.0/24 */, 24, NDPI_PROTOCOL_NETFLIX }, - { 0x2D390000 /* 45.57.0.0/17 */, 17, NDPI_PROTOCOL_NETFLIX }, - { 0x40788000 /* 64.120.128.0/17 */, 17, NDPI_PROTOCOL_NETFLIX }, - { 0x42C58000 /* 66.197.128.0/17 */, 17, NDPI_PROTOCOL_NETFLIX }, - { 0x4535E000 /* 69.53.224.0/19 */, 19, NDPI_PROTOCOL_NETFLIX }, - { 0x6CAF2000 /* 108.175.32.0/20 */, 20, NDPI_PROTOCOL_NETFLIX }, - { 0xB902DC00 /* 185.2.220.0/22 */, 22, NDPI_PROTOCOL_NETFLIX }, - { 0xB909BC00 /* 185.9.188.0/22 */, 22, NDPI_PROTOCOL_NETFLIX }, - { 0xC0AD4000 /* 192.173.64.0/18 */, 18, NDPI_PROTOCOL_NETFLIX }, - { 0xC6266000 /* 198.38.96.0/19 */, 19, NDPI_PROTOCOL_NETFLIX }, - { 0xC62D3000 /* 198.45.48.0/20 */, 20, NDPI_PROTOCOL_NETFLIX }, - { 0xD194D687 /* 209.148.214.135/21*/, 21, NDPI_PROTOCOL_NETFLIX }, - /* OCS GO (Orange Cinéma Séries) */ @@ -598,8 +528,9 @@ static ndpi_network host_protocol_list[] = { /* Teamviewer - AS43304, AS208187, AS208175 - router1.teamviewer.com... router16.teamviewer.com */ + router1.teamviewer.com... router16.teamviewer.com + TODO: are we able to autogenerate this list (like the other Teamviewer IPs)? + */ { 0x25FCE044 /* 37.252.224.68 */, 30, NDPI_PROTOCOL_TEAMVIEWER }, { 0x25FCE048 /* 37.252.224.72 */, 30, NDPI_PROTOCOL_TEAMVIEWER }, { 0x25FCE04C /* 37.252.224.76 */, 31, NDPI_PROTOCOL_TEAMVIEWER }, @@ -672,8 +603,6 @@ static ndpi_network host_protocol_list[] = { { 0xB2FF9BBC /* 178.255.155.188 */, 31, NDPI_PROTOCOL_TEAMVIEWER }, { 0xB2FF9BBE /* 178.255.155.190 */, 32, NDPI_PROTOCOL_TEAMVIEWER }, { 0xB86B0EA8 /* 184.107.14.168 */, 29, NDPI_PROTOCOL_TEAMVIEWER }, - { 0xB9BC2000 /* 185.188.32.0 */, 22, NDPI_PROTOCOL_TEAMVIEWER }, - { 0xB9F51C00 /* 185.245.28.0 */, 22, NDPI_PROTOCOL_TEAMVIEWER }, { 0xBC414C84 /* 188.65.76.132 */, 30, NDPI_PROTOCOL_TEAMVIEWER }, { 0xBC414C88 /* 188.65.76.136 */, 30, NDPI_PROTOCOL_TEAMVIEWER }, { 0xBC414C8C /* 188.65.76.140 */, 31, NDPI_PROTOCOL_TEAMVIEWER }, @@ -722,8 +651,6 @@ static ndpi_network host_protocol_list[] = { { 0xBCACFE88 /* 188.172.254.136 */, 30, NDPI_PROTOCOL_TEAMVIEWER }, { 0xBCACFE8C /* 188.172.254.140 */, 31, NDPI_PROTOCOL_TEAMVIEWER }, { 0xC0AF7958 /* 192.175.121.88 */, 29, NDPI_PROTOCOL_TEAMVIEWER }, - { 0xC120E000 /* 193.32.224.0 */, 22, NDPI_PROTOCOL_TEAMVIEWER }, - { 0xC13A6400 /* 193.58.100.0 */, 22, NDPI_PROTOCOL_TEAMVIEWER }, { 0xD5E3A264 /* 213.227.162.100 */, 30, NDPI_PROTOCOL_TEAMVIEWER }, { 0xD5E3A268 /* 213.227.162.104 */, 29, NDPI_PROTOCOL_TEAMVIEWER }, { 0xD5E3A270 /* 213.227.162.112 */, 29, NDPI_PROTOCOL_TEAMVIEWER }, @@ -790,23 +717,6 @@ static ndpi_network host_protocol_list[] = { { 0xD8DB7000 /* 216.219.112.0 */, 20, NDPI_PROTOCOL_CITRIX }, { 0xADC70000 /* 173.199.0.0 */, 18, NDPI_PROTOCOL_CITRIX }, - /* - Cisco Webex LLC - origin AS13445 - */ - - { 0x0819C000 /* 8.25.192.0/24 */, 24, NDPI_PROTOCOL_WEBEX }, - { 0x3E6DC000 /* 62.109.192.0/18 */, 18, NDPI_PROTOCOL_WEBEX }, - { 0x40446000 /* 64.68.96.0/19 */, 19, NDPI_PROTOCOL_WEBEX }, - { 0x4272A000 /* 66.114.160.0/20 */, 20, NDPI_PROTOCOL_WEBEX }, - { 0x42A32000 /* 66.163.32.0/19 */, 19, NDPI_PROTOCOL_WEBEX }, - { 0x721DC000 /* 114.29.192.0/19 */, 19, NDPI_PROTOCOL_WEBEX }, - { 0xADF30000 /* 173.243.0.0/20 */, 20, NDPI_PROTOCOL_WEBEX }, - { 0xCFB6A000 /* 207.182.160.0/19*/, 19, NDPI_PROTOCOL_WEBEX }, - { 0xD0085100 /* 208.8.81.0/24 */, 24, NDPI_PROTOCOL_WEBEX }, - { 0xD1C5C000 /* 209.197.192.0/19*/, 19, NDPI_PROTOCOL_WEBEX }, - { 0xD204C000 /* 210.4.192.0/20 */, 20, NDPI_PROTOCOL_WEBEX }, - /* Viber Media S.à r.l. https://whois.arin.net/rest/nets;q=AWS-VIBER-MEDIA?showDetails=true&showARIN=false&showNonArinTopLevelNet=false&ext=netref2 @@ -830,17 +740,6 @@ static ndpi_network host_protocol_list[] = { { 0x36DBBFA0 /* 54.219.191.160/28 */, 28, NDPI_PROTOCOL_VIBER }, { 0x36E8FB90 /* 54.232.251.144/28 */, 28, NDPI_PROTOCOL_VIBER }, - /* - Apple Inc. - origin AS714, AS6185 - */ - - { 0x11000000 /* 17.0.0.0/8 */, 8 , NDPI_PROTOCOL_APPLE }, - { 0xC0233200 /* 192.35.50.0/24 */, 24, NDPI_PROTOCOL_APPLE }, - { 0xC6B71000 /* 198.183.16.0/24 */, 24, NDPI_PROTOCOL_APPLE }, - { 0xC6B71100 /* 198.183.17.0/24 */, 24, NDPI_PROTOCOL_APPLE }, - { 0xCDB4AF00 /* 205.180.175.0/24 */, 24, NDPI_PROTOCOL_APPLE }, - /* Dropbox, Inc. origin AS19679 @@ -886,19 +785,6 @@ static ndpi_network host_protocol_list[] = { { 0xA2D52000 /* 162.213.32.0/22 */, 22, NDPI_PROTOCOL_UBUNTUONE }, { 0xB97DBC00 /* 185.125.188.0/22 */, 22, NDPI_PROTOCOL_UBUNTUONE }, - /* - Telegram Messenger LLP - origin: AS62041, AS62014, AS59930 - */ - - { 0x5B6C0400 /* 91.108.4.0/22 */, 22, NDPI_PROTOCOL_TELEGRAM }, - { 0x5B6C0800 /* 91.108.8.0/22 */, 22, NDPI_PROTOCOL_TELEGRAM }, - { 0x5B6C0C00 /* 91.108.12.0/22 */, 22, NDPI_PROTOCOL_TELEGRAM }, - { 0x5B6C1000 /* 91.108.16.0/22 */, 22, NDPI_PROTOCOL_TELEGRAM }, - { 0x5B6C1400 /* 91.108.20.0/22 */, 22, NDPI_PROTOCOL_TELEGRAM }, - { 0x5B6C3800 /* 91.108.56.0/22 */, 22, NDPI_PROTOCOL_TELEGRAM }, - { 0x959AA000 /* 149.154.160.0/20*/, 20, NDPI_PROTOCOL_TELEGRAM }, - /* Twitch Interactive Inc. origin AS46489 diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 825c1597b3d..de989a10599 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -63,6 +63,13 @@ #include "ndpi_ms_skype_teams_match.c.inc" #include "ndpi_google_match.c.inc" #include "ndpi_google_cloud_match.c.inc" +#include "ndpi_asn_telegram.c.inc" +#include "ndpi_asn_apple.c.inc" +#include "ndpi_asn_twitter.c.inc" +#include "ndpi_asn_netflix.c.inc" +#include "ndpi_asn_webex.c.inc" +#include "ndpi_asn_teamviewer.c.inc" +#include "ndpi_asn_facebook.c.inc" /* Third party libraries */ #include "third_party/include/ndpi_patricia.h" @@ -2427,6 +2434,15 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_google_protocol_list); if(!(prefs & ndpi_dont_load_google_cloud_list)) ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_google_cloud_protocol_list); + if(!(prefs & ndpi_dont_load_asn_lists)) { + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_telegram_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_apple_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_twitter_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_netflix_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_webex_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_teamviewer_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_facebook_protocol_list); + } } ndpi_str->ip_risk_mask_ptree = ndpi_patricia_new(32 /* IPv4 */); diff --git a/utils/asn_apple.sh b/utils/asn_apple.sh new file mode 100755 index 00000000000..aa1bd1c19cb --- /dev/null +++ b/utils/asn_apple.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +cd "$(dirname "${0}")" || return + +DEST=../src/lib/ndpi_asn_apple.c.inc +LIST=/tmp/apple.list +LIST_MERGED=/tmp/apple_m.list + +echo "(1) Downloading Apple routes..." +rm -f $LIST +./get_routes_by_asn.sh "AS714" >> $LIST +./get_routes_by_asn.sh "AS6185" >> $LIST +./get_routes_by_asn.sh "AS2709" >> $LIST + +echo "(2) Processing IP addresses..." +./mergeipaddrlist.py $LIST > $LIST_MERGED +./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_APPLE > $DEST +rm -f $LIST $LIST_MERGED + +echo "(3) Apple IPs are available in $DEST" diff --git a/utils/asn_facebook.sh b/utils/asn_facebook.sh new file mode 100755 index 00000000000..a40d3201dd4 --- /dev/null +++ b/utils/asn_facebook.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +cd "$(dirname "${0}")" || return + +DEST=../src/lib/ndpi_asn_facebook.c.inc +LIST=/tmp/facebook.list +LIST_MERGED=/tmp/facebook_m.list + +echo "(1) Downloading Facebook routes..." +rm -f $LIST +./get_routes_by_asn.sh "AS63293" >> $LIST +./get_routes_by_asn.sh "AS54115" >> $LIST +./get_routes_by_asn.sh "AS34825" >> $LIST +./get_routes_by_asn.sh "AS32934" >> $LIST + +echo "(2) Processing IP addresses..." +./mergeipaddrlist.py $LIST > $LIST_MERGED +./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_FACEBOOK > $DEST +rm -f $LIST $LIST_MERGED + +echo "(3) Facebook IPs are available in $DEST" diff --git a/utils/asn_netflix.sh b/utils/asn_netflix.sh new file mode 100755 index 00000000000..d6482e16239 --- /dev/null +++ b/utils/asn_netflix.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +cd "$(dirname "${0}")" || return + +DEST=../src/lib/ndpi_asn_netflix.c.inc +LIST=/tmp/netflix.list +LIST_MERGED=/tmp/netflix_m.list + +echo "(1) Downloading Netflix routes..." +rm -f $LIST +./get_routes_by_asn.sh "AS55095" >> $LIST +./get_routes_by_asn.sh "AS40027" >> $LIST +./get_routes_by_asn.sh "AS394406" >> $LIST +./get_routes_by_asn.sh "AS2906" >> $LIST + +echo "(2) Processing IP addresses..." +./mergeipaddrlist.py $LIST > $LIST_MERGED +./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_NETFLIX > $DEST +rm -f $LIST $LIST_MERGED + +echo "(3) Netflix IPs are available in $DEST" diff --git a/utils/asn_teamviewer.sh b/utils/asn_teamviewer.sh new file mode 100755 index 00000000000..8cebad9dae7 --- /dev/null +++ b/utils/asn_teamviewer.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +cd "$(dirname "${0}")" || return + +DEST=../src/lib/ndpi_asn_teamviewer.c.inc +LIST=/tmp/teamviewer.list +LIST_MERGED=/tmp/teamviewer_m.list + +echo "(1) Downloading Teamviewer routes..." +rm -f $LIST +./get_routes_by_asn.sh "AS43304" >> $LIST +./get_routes_by_asn.sh "AS212710" >> $LIST +./get_routes_by_asn.sh "AS208187" >> $LIST +./get_routes_by_asn.sh "AS208175" >> $LIST + +echo "(2) Processing IP addresses..." +./mergeipaddrlist.py $LIST > $LIST_MERGED +./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_TEAMVIEWER > $DEST +rm -f $LIST $LIST_MERGED + +echo "(3) Teamviewer IPs are available in $DEST" diff --git a/utils/asn_telegram.sh b/utils/asn_telegram.sh new file mode 100755 index 00000000000..3e53675a895 --- /dev/null +++ b/utils/asn_telegram.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +cd "$(dirname "${0}")" || return + +DEST=../src/lib/ndpi_asn_telegram.c.inc +LIST=/tmp/telegram.list +LIST_MERGED=/tmp/telegram_m.list + +echo "(1) Downloading Telegram routes..." +rm -f $LIST +./get_routes_by_asn.sh "AS62041" >> $LIST +./get_routes_by_asn.sh "AS62014" >> $LIST +./get_routes_by_asn.sh "AS59930" >> $LIST +./get_routes_by_asn.sh "AS44907" >> $LIST +./get_routes_by_asn.sh "AS211157" >> $LIST + +echo "(2) Processing IP addresses..." +./mergeipaddrlist.py $LIST > $LIST_MERGED +./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_TELEGRAM > $DEST +rm -f $LIST $LIST_MERGED + +echo "(3) Telegram IPs are available in $DEST" diff --git a/utils/asn_twitter.sh b/utils/asn_twitter.sh new file mode 100755 index 00000000000..20b9d439920 --- /dev/null +++ b/utils/asn_twitter.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +cd "$(dirname "${0}")" || return + +DEST=../src/lib/ndpi_asn_twitter.c.inc +LIST=/tmp/twitter.list +LIST_MERGED=/tmp/twitter_m.list + +echo "(1) Downloading Twitter routes..." +rm -f $LIST +./get_routes_by_asn.sh "AS63179" >> $LIST +./get_routes_by_asn.sh "AS54888" >> $LIST +./get_routes_by_asn.sh "AS35995" >> $LIST +./get_routes_by_asn.sh "AS13414" >> $LIST + +echo "(2) Processing IP addresses..." +./mergeipaddrlist.py $LIST > $LIST_MERGED +./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_TWITTER > $DEST +rm -f $LIST $LIST_MERGED + +echo "(3) Twitter IPs are available in $DEST" diff --git a/utils/asn_webex.sh b/utils/asn_webex.sh new file mode 100755 index 00000000000..f971635f355 --- /dev/null +++ b/utils/asn_webex.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +cd "$(dirname "${0}")" || return + +DEST=../src/lib/ndpi_asn_webex.c.inc +LIST=/tmp/webex.list +LIST_MERGED=/tmp/webex_m.list + +echo "(1) Downloading Webex routes..." +rm -f $LIST +./get_routes_by_asn.sh "AS6577" >> $LIST +./get_routes_by_asn.sh "AS399937" >> $LIST +./get_routes_by_asn.sh "AS16472" >> $LIST +./get_routes_by_asn.sh "AS13445" >> $LIST + +echo "(2) Processing IP addresses..." +./mergeipaddrlist.py $LIST > $LIST_MERGED +./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_WEBEX > $DEST +rm -f $LIST $LIST_MERGED + +echo "(3) Webex IPs are available in $DEST" diff --git a/utils/get_routes_by_asn.sh b/utils/get_routes_by_asn.sh new file mode 100755 index 00000000000..742eaf95763 --- /dev/null +++ b/utils/get_routes_by_asn.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 AS-Number" >&2 + return +fi + +LIST=/tmp/asn.json +ORIGIN="https://stat.ripe.net/data/announced-prefixes/data.json?resource=$1" + +http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN}) +if [ $http_response != "200" ]; then + echo "Error $http_response: wrong ASN number/format?" >&2 + return +fi + +jq -r '.data.prefixes[].prefix' $LIST | grep -v ":" + +rm -f $LIST diff --git a/utils/mergeipaddrlist.py b/utils/mergeipaddrlist.py new file mode 100755 index 00000000000..21ec8bce764 --- /dev/null +++ b/utils/mergeipaddrlist.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 + +import sys +import socket, struct +import netaddr + +if len (sys.argv) == 3: + proto = sys.argv[2] + +if len(sys.argv) < 2: + print("Usage: mergeipaddrlist.py ") + sys.exit (1) + +ipFile = open(sys.argv[1]) +ipAddresses = [i for i in ipFile.readlines()] +ipAddresses = sorted(ipAddresses) +cidrs = netaddr.cidr_merge(ipAddresses) +for cidr in cidrs: + print(cidr) diff --git a/utils/update_every_content_match_lists.sh b/utils/update_every_lists.sh similarity index 69% rename from utils/update_every_content_match_lists.sh rename to utils/update_every_lists.sh index 74c8e9ca253..d8c559a3592 100755 --- a/utils/update_every_content_match_lists.sh +++ b/utils/update_every_lists.sh @@ -1,6 +1,6 @@ #/bin/sh -cd "$(dirname "${0}")" +cd "$(dirname "${0}")" || return ./aws_ip_addresses_download.sh ./azure_ip_addresses_download.sh @@ -12,3 +12,11 @@ cd "$(dirname "${0}")" ./zoom_ip_addresses_download.sh ./google_cloud_ip_addresses_download.sh ./google_ip_addresses_download.sh + +./asn_apple.sh +./asn_facebook.sh +./asn_netflix.sh +./asn_teamviewer.sh +./asn_telegram.sh +./asn_twitter.sh +./asn_webex.sh