diff --git a/scripts/populate_file_web_server.py b/scripts/populate_file_web_server.py new file mode 100644 index 000000000000..9e0ada1f99a1 --- /dev/null +++ b/scripts/populate_file_web_server.py @@ -0,0 +1,319 @@ +import sys +import os +import time +import argparse +from http import HTTPStatus +try: + import requests +except ImportError: + print("requests module is not installed. script will fail to execute") + +#debug print level +PRINT_LEVEL_ERROR = "err" +PRINT_LEVEL_WARN = "warn" +PRINT_LEVEL_INFO = "info" +PRINT_LEVEL_VERBOSE = "verbose" + +PRINT_LEVEL_LUT = {PRINT_LEVEL_ERROR : 1, + PRINT_LEVEL_WARN : 2, + PRINT_LEVEL_INFO : 3, + PRINT_LEVEL_VERBOSE : 4 } + +#return code +RET_CODE_SUCCESS = 0 +RET_CODE_CANNOT_CREATE_FILE = -1 +RET_CODE_CANNOT_OPEN_FILE = -2 +RET_CODE_HTTP_SERVER_ERROR = -3 +RET_CODE_CANNOT_WRITE_FILE = -4 + +#constants +RESOURCES_FILE_NAME = 'versions-web' +EXCLUDE_DIRECTORES = ['fsroot', 'target'] +HASH_SEPARATOR = '-' +DEFAULT_INVALID_INPUT = 'none' + +# global variables +g_current_print_level = PRINT_LEVEL_INFO + +#Script debug features (disabled by default) +g_delete_resources_in_cache = True + + +# global Classes +class Resource: + def __init__(self, line, file): + self.file = file + temp=line.split("==") + assert(2==len(temp)) + self.url=temp[0].strip() + self.hash=temp[1].strip() + temp=self.url.split("/") + assert(len(temp)>0) + self.name=temp[len(temp)-1] + #handle special scenarios + if 0 != self.name.count('?') == True: + temp = self.name.split("?") + self.name = temp[0] + + def get_unique_name(self): + return self.name + HASH_SEPARATOR + self.hash + + def get_url(self): + return self.url + + def __str__(self): + ret_val = "Resource name: " + self.name + "\n" + ret_val += "File: " + self.file + "\n" + ret_val += "Hash: " + self.hash + "\n" + ret_val += "Full URL: " + self.url + return ret_val + + + +# Helper functions + +def info_print(msg): + if PRINT_LEVEL_LUT[g_current_print_level] >= PRINT_LEVEL_LUT[PRINT_LEVEL_INFO]: + print(msg) + +def info_print_in_place(msg): + if PRINT_LEVEL_LUT[g_current_print_level] >= PRINT_LEVEL_LUT[PRINT_LEVEL_INFO]: + print(msg, end='\r') + +def verbose_print(msg): + if PRINT_LEVEL_LUT[g_current_print_level] >= PRINT_LEVEL_LUT[PRINT_LEVEL_VERBOSE]: + print(msg) + +def warn_print(msg): + if PRINT_LEVEL_LUT[g_current_print_level] >= PRINT_LEVEL_LUT[PRINT_LEVEL_WARN]: + print(msg) + +def error_print(msg): + if PRINT_LEVEL_LUT[g_current_print_level] >= PRINT_LEVEL_LUT[PRINT_LEVEL_ERROR]: + print(msg) + +def create_dir_if_not_exist(dir): + if not os.path.exists(dir): + try: + os.mkdir(dir) + except: + warn_print("Cannot create directory " + dir) + +def delete_file_if_exist(file): + + if os.path.exists(file): + try: + os.remove(file) + except: + warn_print("Cannot delete " + file) + +# Logic functions + +def generate_output_file(resources, dest_url_valid, dest_url, output_file_name): + + try: + with open(output_file_name, 'w') as f: + for unique_name in resources.keys(): + resource = resources[unique_name] + if True == dest_url_valid: + line = dest_url + else: + line = resource.get_url() + if line[-1] != '/': + line += '/' + line += resource.name + "==" + resource.hash + f.write(line + '\n') + except: + warn_print(output_file_name + " cannot be created") + return RET_CODE_CANNOT_CREATE_FILE + + f.close() + + return RET_CODE_SUCCESS + +def upload_resource_to_server(resource_path, resource_name, user, key, server_url): + + url_full_path = server_url + "/" + resource_name + + try: + f = open(resource_path, 'rb') + except: + err_print("Cannot open " + resource_path) + return RET_CODE_CANNOT_OPEN_FILE + + headers = {'Content-type': 'application', 'Slug': resource_name} + response = requests.put(url_full_path, data=f, + headers=headers, auth=(user, key)) + + f.close() + + if response.status_code != HTTPStatus.CREATED.value: + err_print(f"HTTP request returned status code {response.status_code}, expected {HTTPStatus.CREATED.value}") + return RET_CODE_HTTP_SERVER_ERROR + + # JSON response empty only when status code is 204 + reported_md5 = response.json().get('checksums', {}).get('md5') + file_md5 = resource_name.split(HASH_SEPARATOR)[-1] + + # Check if server reports checksum, if so compare reported sum and the one + # specified in filename + if reported_md5 != None and reported_md5 != file_md5: + warn_print(f"Server reported file's chsum {reported_md5}, expected {file_md5}") + + + return RET_CODE_SUCCESS + +def download_external_resouce(resource, cache_path): + + resource_path_in_cache = cache_path + os.sep + resource.get_unique_name() + + r = requests.get(resource.get_url(), allow_redirects=True) + + try: + f = open(resource_path_in_cache, 'wb') + f.write(r.content) + f.close() + except: + error_print("Cannot write " + resource_path_in_cache + " to cache") + resource_path_in_cache = "" #report error + + return resource_path_in_cache + +def get_resources_list(resource_files_list): + + resource_list = list() + + for file_name in resource_files_list: + try: + with open(file_name, 'r') as f: + for line in f: + resource_list.append(Resource(line, file_name)) + except: + warn_print(file_name + " cannot be opened") + + return resource_list + +def filter_out_dir(subdir): + ret_val = False + + for exclude in EXCLUDE_DIRECTORES: + if exclude in subdir.split(os.sep): + ret_val = True + break + + return ret_val + +def get_resource_files_list(serach_path): + resource_files_list = list() + + for subdir, dirs, files in os.walk(serach_path): + for file in files: + if False == filter_out_dir(subdir) and RESOURCES_FILE_NAME == file: + file_full_path = os.path.join(subdir, file) + verbose_print("Found resource file :" + file_full_path) + resource_files_list.append(file_full_path) + + return resource_files_list + +def parse_args(): + + parser = argparse.ArgumentParser(description='Various pre-steps for build compilation') + + parser.add_argument('-s', '--source', default=".", + help='Search path for ' + RESOURCES_FILE_NAME + ' files') + + parser.add_argument('-c', '--cache', default="." + os.sep + "tmp", + help='Path to cache for storing content before uploading to server') + + parser.add_argument('-p', '--print', default=PRINT_LEVEL_INFO, + choices=[PRINT_LEVEL_ERROR, PRINT_LEVEL_WARN, PRINT_LEVEL_INFO, PRINT_LEVEL_VERBOSE], + help='Print level verbosity') + + parser.add_argument('-o', '--output', default=DEFAULT_INVALID_INPUT, + help='Output file name to hold the list of packages') + + parser.add_argument('-u', '--user', default=DEFAULT_INVALID_INPUT, + help='User for server authentication') + + parser.add_argument('-k', '--key', default=DEFAULT_INVALID_INPUT, + help='API key server authentication') + + parser.add_argument('-d', '--dest', default=DEFAULT_INVALID_INPUT, + help='URL for destination web file server') + + return parser.parse_args() + +def main(): + + global g_current_print_level + ret_val = RET_CODE_SUCCESS + resource_counter = 0.0 + resource_dict = dict() + + args = parse_args() + + g_current_print_level = args.print + + resource_files_list = get_resource_files_list(args.source) + + resource_list = get_resources_list(resource_files_list) + + #remove duplications + for resource in resource_list: + unique_name = resource.get_unique_name() + if not unique_name in resource_dict.keys(): + resource_dict[unique_name] = resource + + info_print("Found " + str(len(resource_files_list)) + " version files and " + str(len(resource_dict.keys())) + " unique resources") + + if args.dest != DEFAULT_INVALID_INPUT: + upload_files_to_server = True + info_print("Upload files to URL - " + args.dest) + else: + upload_files_to_server = False + info_print("Skipping files upload to server") + + #create cache directory if not exist + create_dir_if_not_exist(args.cache) + + #download content to cache and then upload to web server + for unique_name in resource_dict.keys(): + + resource = resource_dict[unique_name] + + verbose_print(resource) + + resource_counter += 1.0 + + #download content to cache + file_in_cache = download_external_resouce(resource, args.cache) + + if "" == file_in_cache: + return RET_CODE_CANNOT_WRITE_FILE + + if True == upload_files_to_server: + #upload content to web server + ret_val = upload_resource_to_server(file_in_cache, unique_name, args.user, args.key, args.dest) + if ret_val != RET_CODE_SUCCESS: + return ret_val + + if True == g_delete_resources_in_cache: + delete_file_if_exist(file_in_cache) + + info_print_in_place("Downloading Data. Progress " + str(int(100.0*resource_counter/len(resource_dict.keys()))) + "%") #print progress bar + + # generate version output file as needed + if args.output != DEFAULT_INVALID_INPUT: + ret_val = generate_output_file(resource_dict, upload_files_to_server, args.dest, args.output) + info_print("Generate output file " + args.output) + + return ret_val + + +# Entry function +if __name__ == '__main__': + + ret_val = main() + + sys.exit(ret_val) +