From 56d11f62e8ab9458af4ee128d6de510bd4098e62 Mon Sep 17 00:00:00 2001 From: pasabanov Date: Tue, 16 Jul 2024 15:30:25 +0300 Subject: [PATCH] python: reformatted, fixed bugs --- utils/google.py | 65 +++++++++++++++++++++------------------- utils/google6.py | 65 +++++++++++++++++++++------------------- utils/hostname2list.py | 12 ++++---- utils/mergeipaddrlist.py | 8 ++--- 4 files changed, 76 insertions(+), 74 deletions(-) mode change 100644 => 100755 utils/google6.py diff --git a/utils/google.py b/utils/google.py index e8b4114c8f5..c59ebed35b5 100755 --- a/utils/google.py +++ b/utils/google.py @@ -4,38 +4,41 @@ import urllib.request import netaddr -GOOG_URL="https://www.gstatic.com/ipranges/goog.json" -CLOUD_URL="https://www.gstatic.com/ipranges/cloud.json" +GOOG_URL = "https://www.gstatic.com/ipranges/goog.json" +CLOUD_URL = "https://www.gstatic.com/ipranges/cloud.json" + def read_url(url): - try: - s = urllib.request.urlopen(url).read() - return json.loads(s) - except urllib.error.HTTPError: - print("Invalid HTTP response from %s" % url) - return {} - except json.decoder.JSONDecodeError: - print("Could not parse HTTP response from %s" % url) - return {} + try: + s = urllib.request.urlopen(url).read() + return json.loads(s) + except urllib.request.HTTPError: + print("Invalid HTTP response from %s" % url) + return {} + except json.decoder.JSONDecodeError: + print("Could not parse HTTP response from %s" % url) + return {} + def main(): - goog_json=read_url(GOOG_URL) - cloud_json=read_url(CLOUD_URL) - - if goog_json and cloud_json: -# print("{} published: {}".format(GOOG_URL,goog_json.get('creationTime'))) -# print("{} published: {}".format(CLOUD_URL,cloud_json.get('creationTime'))) - goog_cidrs = netaddr.IPSet() - for pref in goog_json['prefixes']: - if pref.get('ipv4Prefix'): - goog_cidrs.add(pref.get('ipv4Prefix')) - cloud_cidrs = netaddr.IPSet() - for pref in cloud_json['prefixes']: - if pref.get('ipv4Prefix'): - cloud_cidrs.add(pref.get('ipv4Prefix')) -# print("IP ranges for Google APIs and services default domains:") - for i in goog_cidrs.difference(cloud_cidrs).iter_cidrs(): - print(i) - -if __name__=='__main__': - main() + goog_json = read_url(GOOG_URL) + cloud_json = read_url(CLOUD_URL) + + if goog_json and cloud_json: + # print("{} published: {}".format(GOOG_URL,goog_json.get('creationTime'))) + # print("{} published: {}".format(CLOUD_URL,cloud_json.get('creationTime'))) + goog_cidrs = netaddr.IPSet() + for pref in goog_json['prefixes']: + if pref.get('ipv4Prefix'): + goog_cidrs.add(pref.get('ipv4Prefix')) + cloud_cidrs = netaddr.IPSet() + for pref in cloud_json['prefixes']: + if pref.get('ipv4Prefix'): + cloud_cidrs.add(pref.get('ipv4Prefix')) + # print("IP ranges for Google APIs and services default domains:") + for i in goog_cidrs.difference(cloud_cidrs).iter_cidrs(): + print(i) + + +if __name__ == '__main__': + main() diff --git a/utils/google6.py b/utils/google6.py old mode 100644 new mode 100755 index 2d9ffa27ca0..a09580a104b --- a/utils/google6.py +++ b/utils/google6.py @@ -4,38 +4,41 @@ import urllib.request import netaddr -GOOG_URL="https://www.gstatic.com/ipranges/goog.json" -CLOUD_URL="https://www.gstatic.com/ipranges/cloud.json" +GOOG_URL = "https://www.gstatic.com/ipranges/goog.json" +CLOUD_URL = "https://www.gstatic.com/ipranges/cloud.json" + def read_url(url): - try: - s = urllib.request.urlopen(url).read() - return json.loads(s) - except urllib.error.HTTPError: - print("Invalid HTTP response from %s" % url) - return {} - except json.decoder.JSONDecodeError: - print("Could not parse HTTP response from %s" % url) - return {} + try: + s = urllib.request.urlopen(url).read() + return json.loads(s) + except urllib.request.HTTPError: + print("Invalid HTTP response from %s" % url) + return {} + except json.decoder.JSONDecodeError: + print("Could not parse HTTP response from %s" % url) + return {} + def main(): - goog_json=read_url(GOOG_URL) - cloud_json=read_url(CLOUD_URL) - - if goog_json and cloud_json: -# print("{} published: {}".format(GOOG_URL,goog_json.get('creationTime'))) -# print("{} published: {}".format(CLOUD_URL,cloud_json.get('creationTime'))) - goog_cidrs = netaddr.IPSet() - for pref in goog_json['prefixes']: - if pref.get('ipv6Prefix'): - goog_cidrs.add(pref.get('ipv6Prefix')) - cloud_cidrs = netaddr.IPSet() - for pref in cloud_json['prefixes']: - if pref.get('ipv6Prefix'): - cloud_cidrs.add(pref.get('ipv6Prefix')) -# print("IP ranges for Google APIs and services default domains:") - for i in goog_cidrs.difference(cloud_cidrs).iter_cidrs(): - print(i) - -if __name__=='__main__': - main() + goog_json = read_url(GOOG_URL) + cloud_json = read_url(CLOUD_URL) + + if goog_json and cloud_json: + # print("{} published: {}".format(GOOG_URL,goog_json.get('creationTime'))) + # print("{} published: {}".format(CLOUD_URL,cloud_json.get('creationTime'))) + goog_cidrs = netaddr.IPSet() + for pref in goog_json['prefixes']: + if pref.get('ipv6Prefix'): + goog_cidrs.add(pref.get('ipv6Prefix')) + cloud_cidrs = netaddr.IPSet() + for pref in cloud_json['prefixes']: + if pref.get('ipv6Prefix'): + cloud_cidrs.add(pref.get('ipv6Prefix')) + # print("IP ranges for Google APIs and services default domains:") + for i in goog_cidrs.difference(cloud_cidrs).iter_cidrs(): + print(i) + + +if __name__ == '__main__': + main() diff --git a/utils/hostname2list.py b/utils/hostname2list.py index d466b015d2f..beb078a1f87 100755 --- a/utils/hostname2list.py +++ b/utils/hostname2list.py @@ -1,19 +1,17 @@ #!/usr/bin/env python3 import sys -import socket, struct # This scripts is used to create "hostname/sni -> protocols" lists. if len(sys.argv) < 6: print("Usage: {} ".format(sys.argv[0])) - sys.exit (1) + sys.exit(1) name = sys.argv[2] proto = sys.argv[3] category = sys.argv[4] breed = sys.argv[5] - print("""/* * * This file is generated automatically and part of nDPI @@ -37,23 +35,23 @@ """) -print("static ndpi_protocol_match "+proto.lower()+"_hostname_list[] = {") +print("static ndpi_protocol_match " + proto.lower() + "_hostname_list[] = {") lines = 0 with open(sys.argv[1]) as fp: for cnt, line in enumerate(fp): line = line.rstrip() - if(line != ""): + if line != "": lines += 1 x = line.split("/") - if(len(x) == 2): + if len(x) == 2: host = x[0] else: host = line - if(host != ""): + if host != "": print(' { ' + f'"{host}", "{name}", {proto}, {category}, {breed}, NDPI_PROTOCOL_DEFAULT_LEVEL' + ' },') print(" /* End */") diff --git a/utils/mergeipaddrlist.py b/utils/mergeipaddrlist.py index 9fced560010..2248f1a7edf 100755 --- a/utils/mergeipaddrlist.py +++ b/utils/mergeipaddrlist.py @@ -1,20 +1,18 @@ #!/usr/bin/env python3 import sys -import socket -import struct import netaddr -if len (sys.argv) == 3: +if len(sys.argv) == 3: proto = sys.argv[2] if len(sys.argv) < 2: print("Usage: mergeipaddrlist.py ") - sys.exit (1) + sys.exit(1) ipFile = open(sys.argv[1]) ipAddresses = list(ipFile.readlines()) -ipAddresses = [x.replace("\n","") for x in ipAddresses] +ipAddresses = [x.replace("\n", "") for x in ipAddresses] ipAddresses = sorted(ipAddresses) cidrs = netaddr.cidr_merge(ipAddresses)