diff --git a/CI/build/.flake8 b/CI/build/.flake8 index 9b7e20c3a6..c5d8607073 100644 --- a/CI/build/.flake8 +++ b/CI/build/.flake8 @@ -1,2 +1,3 @@ [flake8] -max-line-length = 95 +max-line-length = 90 +ignore = E402 diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index f13ac2651e..8063a88d94 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -2,47 +2,54 @@ import collections import concurrent.futures from datetime import timedelta +from glob import glob import json import os from packaging import version +from pathlib import Path +import random import re -import shutil import subprocess import sys import tempfile import textwrap import time +script_path = Path(__file__).parent.resolve() +sys.path.append(str(script_path.parent)) +from utils import createFolder, deleteFolder + if sys.platform.startswith("win32"): from colorama import init init(autoreset=True) -home = os.path.expanduser("~") -tempdir = tempfile.gettempdir() -build_id = time.strftime("_%Y-%m-%d_%H-%M-%S") -script_path = os.path.dirname(os.path.abspath(__file__)) -path_config_filename = os.path.join(script_path, "path_config.json") +home = Path.home() +tempdir = Path(tempfile.gettempdir()) +build_id = time.strftime(".%Y-%m-%d_%H-%M-%S") +path_config_filename = script_path / "path_config.json" -arduino_cli_path = "" -stm32_url = "https://github.com/stm32duino/BoardManagerFiles/raw/main/package_stmicroelectronics_index.json" +arduino_cli_path = Path("") +stm32_url_base = "https://github.com/stm32duino/BoardManagerFiles/raw/main/" +stm32_url = f"{stm32_url_base}package_stmicroelectronics_index.json" sketches_path_list = [] -default_build_output_dir = os.path.join(tempdir, "build_arduinoCliOutput") -build_output_dir = os.path.join(tempdir, "build_arduinoCliOutput" + build_id) -build_output_cache_dir = os.path.join(build_output_dir, "cache") -root_output_dir = os.path.join(home, "Documents", "arduinoCliOutput") +search_path_list = [] +default_build_output_dir = tempdir / "build_arduinoCliOutput" +build_output_dir = tempdir / f"build_arduinoCliOutput{build_id}" +build_output_cache_dir = build_output_dir / "cache" +root_output_dir = home / "Documents" / "arduinoCliOutput" -output_dir = "" -log_file = "" +output_dir = Path("") +log_file = Path("") -# Ouput directory path +# Output directory path bin_dir = "binaries" # Default -sketch_default = os.path.join(script_path, "examples", "BareMinimum") -exclude_file_default = os.path.join(script_path, "conf", "exclude_list.txt") -cores_config_file_default = os.path.join(script_path, "conf", "cores_config.json") -cores_config_file_ci = os.path.join(script_path, "conf", "cores_config_ci.json") +sketch_default = script_path / "examples" / "BareMinimum" +exclude_file_default = script_path / "conf" / "exclude_list.txt" +cores_config_file_default = script_path / "conf" / "cores_config.json" +cores_config_file_ci = script_path / "conf" / "cores_config_ci.json" maintainer_default = "STMicroelectronics" arch_default = "stm32" @@ -53,8 +60,8 @@ arch = arch_default arduino_platform = arduino_platform_default arduino_cli = "" -arduino_cli_default_version = "0.18.0" -arduino_cli_version = arduino_cli_default_version +arduino_cli_default_ver = "0.19.0" +arduino_cli_ver = arduino_cli_default_ver # List sketch_list = [] @@ -81,25 +88,28 @@ fail_count = 0 skip_count = 0 +# error or fatal error +fork_pattern = re.compile(r"^Error during build: fork/exec") +error_pattern = re.compile(r":\d+:\d+:\s.*error:\s|^Error:") +ld_pattern = re.compile("arm-none-eabi/bin/ld:") +overflow_pattern = re.compile( + r"(will not fit in |section .+ is not within )?region( .+ overflowed by [\d]+ bytes)?" +) + # format build_format_header = "| {:^8} | {:42} | {:^10} | {:^7} |" build_format_result = "| {:^8} | {:42} | {:^19} | {:^6.2f}s |" build_separator = "-" * 80 +fsucc = "\033[32msucceeded\033[0m" +ffail = "\033[31mfailed\033[0m" +fskip = "\033[33mskipped\033[0m" +nl = "\n" - -# Create a folder if not exists -def createFolder(folder): - try: - if not os.path.exists(folder): - os.makedirs(folder) - except OSError: - print("Error: Creating directory. " + folder) - - -# Delete targeted folder recursively -def deleteFolder(folder): - if os.path.isdir(folder): - shutil.rmtree(folder, ignore_errors=True) +# index build configuration +idx_b_name = 0 +idx_build = 1 +idx_log = 2 +idx_cmd = 3 def cat(file): @@ -112,17 +122,15 @@ def cat(file): def create_output_log_tree(): # Log output file with open(log_file, "w") as file: - file.write(build_separator + "\nStarts ") + file.write(f"{build_separator}\nStarts ") file.write(time.strftime("%A %d %B %Y %H:%M:%S ")) - file.write( - "\nLog will be available here:\n{}\n".format(os.path.abspath(output_dir)) - ) - file.write(build_separator + "\n") + file.write(f"\nLog will be available here:\n{output_dir.resolve()}\n") + file.write(f"{build_separator}\n") # Folders for board in board_fqbn: - createFolder(os.path.join(output_dir, board, bin_dir)) - createFolder(os.path.join(output_dir, board)) - createFolder(os.path.join(build_output_dir, board)) + createFolder(output_dir / board / bin_dir) + createFolder(output_dir / board) + createFolder(build_output_dir / board) def create_config(): @@ -131,19 +139,16 @@ def create_config(): global build_output_dir global root_output_dir # Create a Json file for a better path management - print( - "'{}' file created. Please check the configuration.".format( - path_config_filename - ) - ) + print(f"'{path_config_filename}' file created.") + print("Please check the configuration.") path_config_file = open(path_config_filename, "w") path_config_file.write( json.dumps( { - "ARDUINO_CLI_PATH": arduino_cli_path, - "SKETCHES_PATHS": sketches_path_list, - "BUILD_OUPUT_DIR": default_build_output_dir, - "ROOT_OUPUT_DIR": root_output_dir, + "ARDUINO_CLI_PATH": str(arduino_cli_path), + "SKETCHES_PATHS": [str(fp) for fp in sketches_path_list], + "BUILD_OUPUT_DIR": str(default_build_output_dir), + "ROOT_OUPUT_DIR": str(root_output_dir), }, indent=2, ) @@ -154,9 +159,10 @@ def create_config(): def check_config(): global arduino_cli - global arduino_cli_version + global arduino_cli_ver global arduino_cli_path global sketches_path_list + global search_path_list global build_output_dir global root_output_dir global output_dir @@ -164,74 +170,119 @@ def check_config(): global stm32_url if args.ci is False: - if os.path.isfile(path_config_filename): + if path_config_filename.is_file(): try: path_config_file = open(path_config_filename, "r") path_config = json.load(path_config_file) # Common path - arduino_cli_path = path_config["ARDUINO_CLI_PATH"] - sketches_path_list = path_config["SKETCHES_PATHS"] - build_output_dir = path_config["BUILD_OUPUT_DIR"] + build_id - root_output_dir = path_config["ROOT_OUPUT_DIR"] + arduino_cli_path = Path(path_config["ARDUINO_CLI_PATH"]) + sketches_path_list = [Path(fp) for fp in path_config["SKETCHES_PATHS"]] + build_output_dir = Path(path_config["BUILD_OUPUT_DIR"]) + build_output_dir = build_output_dir.with_suffix(build_id) + root_output_dir = Path(path_config["ROOT_OUPUT_DIR"]) path_config_file.close() except IOError: print("Failed to open " + path_config_file) else: create_config() - output_dir = os.path.join(root_output_dir, "build" + build_id) - log_file = os.path.join(output_dir, "build_result.log") + output_dir = root_output_dir / f"build{build_id}" + log_file = output_dir / "build_result.log" - if arduino_cli_path != "": - assert os.path.exists( - arduino_cli_path - ), "Path does not exist: {} . Please check the path in the json config file".format( - arduino_cli_path + if arduino_cli_path != Path(): + assert arduino_cli_path.exists(), ( + f"Path {arduino_cli_path} does not exist." + f" Please check path in the json config file." ) if sys.platform.startswith("win32"): - arduino_cli = os.path.join(arduino_cli_path, "arduino-cli.exe") - elif sys.platform.startswith("linux"): - arduino_cli = os.path.join(arduino_cli_path, "arduino-cli") - elif sys.platform.startswith("darwin"): - arduino_cli = os.path.join(arduino_cli_path, "arduino-cli") + arduino_cli = str(arduino_cli_path / "arduino-cli.exe") else: - arduino_cli = "arduino-cli" + arduino_cli = str(arduino_cli_path / "arduino-cli") + # Check arduino-cli version try: output = subprocess.check_output( - [arduino_cli, "version"], stderr=subprocess.STDOUT, + [arduino_cli, "version"], + stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as e: - print('"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode)) + print(f"'{' '.join(e.cmd)}' failed with code: {e.returncode}!") print(e.stdout.decode("utf-8")) quit(e.returncode) else: res = re.match(r".*Version:\s+(\d+\.\d+\.\d+).*", output.decode("utf-8")) if res: - arduino_cli_version = res.group(1) - print("Arduino CLI version used: " + arduino_cli_version) + arduino_cli_ver = res.group(1) + print(f"Arduino CLI version used: {arduino_cli_ver}") + if version.parse(arduino_cli_ver) < version.parse(arduino_cli_default_ver): + print(f"Arduino CLI version < {arduino_cli_default_ver} not supported") else: - print( - "Unable to define Arduino CLI version, use default: " - + arduino_cli_default_version - ) + print("Unable to define Arduino CLI version.") + print(f"Use default: {arduino_cli_default_ver}") if args.url: stm32_url = args.url + # Ensure a configuration exists try: + print("Check/update arduino-cli configuration") + # Try to create configuration file output = subprocess.check_output( - [arduino_cli, "core", "search", "stm32", "--additional-urls", stm32_url], + [arduino_cli, "config", "init", "--additional-urls", stm32_url], + stderr=subprocess.STDOUT, + ).decode("utf-8") + except subprocess.CalledProcessError: + try: + output = subprocess.check_output( + [ + arduino_cli, + "config", + "dump", + ], + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as e: + print(f"'{' '.join(e.cmd)}' failed with code: {e.returncode}!") + print(e.stdout.decode("utf-8")) + quit(e.returncode) + else: + # Check if url is already part of the arduino-cli config + if stm32_url not in output.decode("utf-8"): + # Add it to the config + try: + output = subprocess.check_output( + [ + arduino_cli, + "config", + "add", + "board_manager.additional_urls", + stm32_url, + ], + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as e: + print(f"'{' '.join(e.cmd)}' failed with code: {e.returncode}!") + print(e.stdout.decode("utf-8")) + quit(e.returncode) + # Check if requested platform is installed + try: + output = subprocess.check_output( + [ + arduino_cli, + "core", + "search", + "stm32", + ], stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as e: - print('"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode)) + print(f"'{' '.join(e.cmd)}' failed with code: {e.returncode}!") print(e.stdout.decode("utf-8")) quit(e.returncode) else: if arduino_platform not in output.decode("utf-8"): - print(arduino_platform + " is not installed!") + print(f"{arduino_platform} is not installed!") quit(1) # Add core and library path to sketches_path_list try: @@ -240,24 +291,29 @@ def check_config(): stderr=subprocess.STDOUT, ).decode("utf-8") except subprocess.CalledProcessError as e: - print( - '"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode) - ) + print(f"'{' '.join(e.cmd)}' failed with code: {e.returncode}!") print(e.stdout.decode("utf-8")) quit(e.returncode) else: cli_config = json.loads(output) if cli_config is not None: if cli_config["directories"]["data"] is not None: - sketches_path_list.append(cli_config["directories"]["data"]) + sketches_path_list.append(Path(cli_config["directories"]["data"])) else: print("No data directory") quit(1) if cli_config["directories"]["user"] is not None: - sketches_path_list.append(cli_config["directories"]["user"]) + sketches_path_list.append(Path(cli_config["directories"]["user"])) else: print("No user directory!") quit(1) + # Fill search_path_list to avoid search on the same path + sorted_spl = sorted(set(sketches_path_list)) + search_path_list = [] + while sorted_spl: + p = sorted_spl.pop(0) + if not any(root in p.parents for root in search_path_list): + search_path_list.append(Path(p)) else: print("No arduino-cli config!") quit(1) @@ -269,9 +325,7 @@ def load_core_config(): global arch cores_config_filename = "" if args.config: - assert os.path.exists( - args.config - ), "User core configuration JSON file does not exist" + assert args.config.exists(), f"{args.config} not found" cores_config_filename = args.config else: if args.ci: @@ -291,35 +345,23 @@ def load_core_config(): if arch == core["architecture"]: core_config = core maintainer = core["maintainer"] - print( - "Build configuration for '" - + maintainer - + "' maintainer and '" - + arch - + "' architecture" - ) + print("Build configuration for:") + print(f"- '{maintainer}' maintainer") + print(f"- '{arch}' architecture") break else: - print( - "Core architecture '" + arch + "' not found in " + cores_config_filename - ) + print(f"Core architecture '{arch}' not found in:") + print(f"{cores_config_filename}") arch = arch_default core_config = None except IOError: - print( - "Can't open {} file. Build configuration will not be used.".format( - cores_config_filename - ) - ) + print(f"Can't open {cores_config_filename} file.") + print("Build configuration will not be used.") finally: if core_config is None: - print( - "Using default configuration for '" - + maintainer_default - + "' maintainer and '" - + arch_default - + "' architecture" - ) + print("Using default configuration for:") + print(f"- '{maintainer_default}' maintainer") + print(f"- '{arch_default}' architecture") # Board list have to be initialized before call this function @@ -359,81 +401,83 @@ def parse_core_config(): # print("{}: {}\n".format(key, value)) -def manage_exclude_list(file): - with open(file, "r") as f: +def manage_exclude_list(exfile): + with open(exfile, "r") as f: for line in f.readlines(): if line.rstrip(): exclude_list.append(line.rstrip()) if exclude_list: for pattern in exclude_list: - exclude_pattern = re.compile(".*" + pattern + ".*", re.IGNORECASE) + exclude_pattern = re.compile(f".*{pattern}.*", re.IGNORECASE) for s in reversed(sketch_list): - if exclude_pattern.search(s): + if exclude_pattern.search(str(s)): sketch_list.remove(s) # Manage sketches list def manage_inos(): - # Find all inos or all patterned inos if args.all or args.sketches or args.list == "sketch": find_inos() if args.exclude: - assert os.path.exists(args.exclude), "Excluded list file does not exist" - manage_exclude_list(args.exclude) - elif os.path.exists(exclude_file_default): + exclude_file = Path(args.exclude) + assert exclude_file.exists(), "Excluded list file does not exist" + manage_exclude_list(exclude_file) + elif exclude_file_default.exists(): manage_exclude_list(exclude_file_default) # Only one sketch elif args.ino: - if os.path.exists(args.ino): + ino_file = Path(args.ino) + if ino_file.exists(): # Store only the path - if os.path.isfile(args.ino): - sketch_list.append(os.path.dirname(args.ino)) + if ino_file.is_file(args.ino): + sketch_list.append(ino_file.parent) else: sketch_list.append(args.ino) else: for path in sketches_path_list: - fp = os.path.join(path, args.ino) - if os.path.exists(fp): + fp = path / ino_file + if fp.exists(): # Store only the path - if os.path.isfile(fp): - sketch_list.append(os.path.dirname(fp)) + if fp.is_file(): + sketch_list.append(fp.parent) else: sketch_list.append(fp) break else: - print("Sketch {} path does not exist!".format(args.ino)) + print(f"Sketch {args.ino} path does not exist!") quit(1) # Sketches listed in a file elif args.file: - assert os.path.exists(args.file), "Sketches list file does not exist" - with open(args.file, "r") as f: + sketches_files = Path(args.file) + assert sketches_files.exists(), f"{sketches_files} not found" + with open(sketches_files, "r") as f: for line in f.readlines(): if line.rstrip(): - ino = line.rstrip() - if os.path.exists(ino): + ino = Path(line.rstrip()) + if ino.exists(): # Store only the path - if os.path.isfile(ino): - sketch_list.append(os.path.dirname(ino)) + if ino.is_file(): + sketch_list.append(ino.parent) else: sketch_list.append(ino) else: for path in sketches_path_list: - fp = os.path.join(path, ino) - if os.path.exists(fp): + fp = path / ino + if fp.exists(): # Store only the path - if os.path.isfile(fp): - sketch_list.append(os.path.dirname(fp)) + if fp.is_file(): + sketch_list.append(fp.parent) else: sketch_list.append(fp) break else: - print("Ignore {} as it does not exist.".format(ino)) + print(f"Ignore {ino} as it does not exist.") # Default sketch to build else: sketch_list.append(sketch_default) if len(sketch_list) == 0: - print("No sketch to build for " + arduino_platform + "!") + print(f"No sketch to build for {arduino_platform}!") quit(1) @@ -443,18 +487,28 @@ def find_inos(): # key: path, value: name if args.sketches: arg_sketch_pattern = re.compile(args.sketches, re.IGNORECASE) - for path in sketches_path_list: - for root, dirs, files in os.walk(path, followlinks=True): - for file in files: - if file.endswith((".ino", ".pde")): - if args.sketches: - if arg_sketch_pattern.search(os.path.join(root, file)) is None: - continue - sketch_list.append(root) + for spath in search_path_list: + # Due to issue with Path.glob() which does not follow symlink + # use glob.glob + # See: https://bugs.python.org/issue33428 + # for spath_object in spath.glob("**/*.[ip][nd][oe]"): + # if args.sketches: + # if arg_sketch_pattern.search(str(spath_object)) is None: + # continue + # if spath_object.is_file(): + # sketch_list.append(spath_object.parent) + for sk_ino in glob(str(spath / "**" / "*.[ip][nd][oe]"), recursive=True): + if args.sketches: + if arg_sketch_pattern.search(sk_ino) is None: + continue + p_ino = Path(sk_ino) + if p_ino.is_file(): + sketch_list.append(p_ino.parent) sketch_list = sorted(set(sketch_list)) -# Return a list of all board using the arduino-cli for the specified architecture +# Return a list of all board using the arduino-cli for the specified +# architecture def find_board(): global board_fqbn board_found = {} @@ -462,87 +516,73 @@ def find_board(): if args.board: arg_board_pattern = re.compile(args.board, re.IGNORECASE) - if version.parse(arduino_cli_version) >= version.parse("0.18.0"): - fqbn_key = "fqbn" - else: - fqbn_key = "FQBN" - fqbn_list_tmp = [] try: output = subprocess.check_output( - [arduino_cli, "board", "listall", "--format", "json"], + [arduino_cli, "board", "search", arduino_platform, "--format", "json"], stderr=subprocess.STDOUT, ).decode("utf-8") except subprocess.CalledProcessError as e: - print('"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode)) + print(f"'{' '.join(e.cmd)}' failed with code: {e.returncode}!") print(e.stdout.decode("utf-8")) quit(e.returncode) else: - boards_list = json.loads(output) - if boards_list is not None: - for board in boards_list["boards"]: - if arduino_platform in board[fqbn_key]: - fqbn_list_tmp.append(board[fqbn_key]) + fqbn_list_tmp = [board["fqbn"] for board in json.loads(output)] if not len(fqbn_list_tmp): - print("No boards found for " + arduino_platform) + print(f"No boards found for {arduino_platform}") quit(1) # For STM32 core, pnum is requested for fqbn in fqbn_list_tmp: try: output = subprocess.check_output( - [arduino_cli, "board", "details", "--additional-urls", stm32_url, "--format", "json", fqbn], + [ + arduino_cli, + "board", + "details", + "--format", + "json", + "-b", + fqbn, + ], stderr=subprocess.STDOUT, ).decode("utf-8") except subprocess.CalledProcessError as e: - print( - '"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode) - ) + print(f"'{' '.join(e.cmd)}' failed with code: {e.returncode}!") print(e.stdout.decode("utf-8")) quit(e.returncode) else: board_detail = json.loads(output) - if board_detail is not None: - if "config_options" not in board_detail: - print("No config_options found for " + fqbn) - quit(1) - for option in board_detail["config_options"]: - if option["option"] == "pnum": - for value in option["values"]: - if args.board: - if arg_board_pattern.search(value["value"]) is None: - continue - board_found[value["value"]] = ( - fqbn + ":pnum=" + value["value"] - ) - break - else: - print('No detail found for:"' + fqbn + '"!') + for val in next( + ( + item + for item in board_detail.get("config_options", []) + if item["option"] == "pnum" + ), + {}, + ).get("values", []): + if args.board: + if arg_board_pattern.search(val["value"]) is None: + continue + board_found[val["value"]] = f"{fqbn}:pnum={val['value']}" if board_found: board_fqbn = collections.OrderedDict(sorted(board_found.items())) else: - print("No board found for " + arduino_platform + "!") + print(f"No board found for {arduino_platform}!") quit(1) # Check the status -def check_status(status, build_conf, boardKo): +def check_status(status, build_conf, boardKo, nb_build_conf): global nb_build_passed global nb_build_failed - sketch_name = os.path.basename(build_conf[4][-1]) + sketch_name = build_conf[idx_cmd][-1].name if status[1] == 0: - result = "\033[32msucceeded\033[0m" + result = fsucc nb_build_passed += 1 elif status[1] == 1: # Check if failed due to a region overflowed - logFile = os.path.join(build_conf[3], sketch_name + ".log") - # error or fatal error - fork_pattern = re.compile(r"^Error during build: fork/exec") - error_pattern = re.compile(r":\d+:\d+:\s.*error:\s|^Error:") - ld_pattern = re.compile("arm-none-eabi/bin/ld:") - overflow_pattern = re.compile( - r"(will not fit in |section .+ is not within )?region( .+ overflowed by [\d]+ bytes)?" - ) + logFile = build_conf[idx_log] / f"{sketch_name}.log" error_found = False for i, line in enumerate(open(logFile)): if error_pattern.search(line): @@ -554,8 +594,8 @@ def check_status(status, build_conf, boardKo): if overflow_pattern.search(line) is None: error_found = True if error_found: - result = "\033[31mfailed\033[0m" - boardKo.append(build_conf[0]) + result = ffail + boardKo.append(build_conf[idx_b_name]) if args.ci: cat(logFile) nb_build_failed += 1 @@ -566,11 +606,12 @@ def check_status(status, build_conf, boardKo): nb_build_passed += 1 else: result = "\033[31merror\033[0m" - + boardKo.append(build_conf[idx_b_name]) + nb_build_failed += 1 print( - (build_format_result).format( - "{}/{}".format(build_conf[1], build_conf[2]), - build_conf[0], + f"{build_format_result}".format( + f"{build_conf[idx_build]}/{nb_build_conf}", + build_conf[idx_b_name], result, status[0], ) @@ -579,148 +620,95 @@ def check_status(status, build_conf, boardKo): # Log sketch build result def log_sketch_build_result(sketch, boardKo, boardSkipped): + nb_ok = len(board_fqbn) - len(boardKo) - len(boardSkipped) + nb_ko = len(boardKo) + nb_na = len(boardSkipped) # Log file with open(log_file, "a") as f: f.write(build_separator + "\n") - f.write( - "Sketch: {} ({}/{})\n".format( - os.path.basename(sketch), - sketch_list.index(sketch) + 1, - len(sketch_list), - ) - ) - f.write(os.path.dirname(sketch)) - f.write( - "\n{} {}, {} {}, {} {}\n".format( - len(board_fqbn) - len(boardKo) - len(boardSkipped), - "succeeded", - len(boardKo), - "failed", - len(boardSkipped), - "skipped", - ) - ) + bcounter = f"{sketch_list.index(sketch) + 1}/{len(sketch_list)}" + f.write(f"Sketch: {sketch.name} ({bcounter})\n") + f.write(str(sketch.parent)) + f.write(f"\n{nb_ok} succeeded, {nb_ko} failed, {nb_na} skipped\n") if len(boardKo): - f.write( - "Failed boards :\n" - + "\n".join( - textwrap.wrap(", ".join(boardKo), 80, break_long_words=False) - ) - ) + sKo = ", ".join(boardKo) + wKo = textwrap.wrap(sKo, 80, break_long_words=False) + f.write(f"Failed boards :\n{nl.join(wKo)}") # f.write("Failed boards :\n" + "\n".join(boardKo)) f.write("\n") if len(boardSkipped): - f.write( - "Skipped boards :\n" - + "\n".join( - textwrap.wrap(", ".join(boardSkipped), 80, break_long_words=False) - ) - ) + sskipped = ", ".join(boardSkipped) + wskipped = textwrap.wrap(sskipped, 80, break_long_words=False) + f.write(f"Skipped boards :\n{nl.join(wskipped)}") # f.write("Skipped boards :\n" + "\n".join(boardSkipped)) f.write("\n") - f.write(build_separator + "\n") + f.write(f"{build_separator}\n") # Standard output print(build_separator) - print( - "Build Summary: {} {}, {} {}, {} {}".format( - len(board_fqbn) - len(boardKo) - len(boardSkipped), - "\033[32msucceeded\033[0m", - len(boardKo), - "\033[31mfailed\033[0m", - len(boardSkipped), - "\033[33mskipped\033[0m", - ) - ) + + print(f"Build Summary: {nb_ok} {fsucc}, {nb_ko} {ffail}, {nb_na} {fskip}") print(build_separator) # Log final result def log_final_result(): # Also equal to len(board_fqbn) * len(sketch_list) - nb_build_total = nb_build_passed + nb_build_failed + nb_build_skipped + nb_build_total = nb_build_passed + nb_build_failed stat_passed = round(nb_build_passed * 100.0 / nb_build_total, 2) stat_failed = round(nb_build_failed * 100.0 / nb_build_total, 2) - stat_skipped = round(nb_build_skipped * 100.0 / nb_build_total, 2) duration = str(timedelta(seconds=time.time() - full_buildTime)) # Log file with open(log_file, "a") as f: - f.write("\n" + build_separator + "\n") + f.write(f"\n{build_separator}\n") f.write("| {:^76} |\n".format("Build summary")) f.write(build_separator + "\n") - f.write( - "{} {} ({}%), {} {} ({}%), {} {} ({}%) of {} builds\n".format( - nb_build_passed, - "succeeded", - stat_passed, - nb_build_failed, - "failed", - stat_failed, - nb_build_skipped, - "skipped", - stat_skipped, - nb_build_total, - ) - ) - f.write("Ends ") - f.write(time.strftime("%A %d %B %Y %H:%M:%S\n")) - f.write("Duration: ") - f.write(duration) - f.write("\nLogs are available here:\n") - f.write(output_dir + "\n") - f.write(build_separator + "\n\n") + ssucc = f"{nb_build_passed} succeeded ({stat_passed}%)" + sfail = f"{nb_build_failed} failed ({stat_failed}%)" + sskip = f"{nb_build_skipped} skipped)" + f.write(f"{ssucc}, {sfail} of {nb_build_total} builds ({sskip})\n") + f.write(f"Ends {time.strftime('%A %d %B %Y %H:%M:%S')}\n") + f.write(f"Duration: {duration}\n") + f.write(f"Logs are available here:\n{output_dir}\n") + f.write(f"{build_separator}\n\n") # Standard output - print( - "Builds Summary: {} {} ({}%), {} {} ({}%), {} {} ({}%) of {} builds".format( - nb_build_passed, - "\033[32msucceeded\033[0m", - stat_passed, - nb_build_failed, - "\033[31mfailed\033[0m", - stat_failed, - nb_build_skipped, - "\033[33mskipped\033[0m", - stat_skipped, - nb_build_total, - ) - ) - print("Duration: " + duration) + ssucc = f"{nb_build_passed} {fsucc} ({stat_passed}%)" + sfail = f"{nb_build_failed} {ffail} ({stat_failed}%)" + sskip = f"{nb_build_skipped} {fskip}" + print(f"Builds Summary: {ssucc}, {sfail} of {nb_build_total} builds ({sskip})") + print(f"Duration: {duration}") print("Logs are available here:") print(output_dir) -# Set up specific options to customise arduino builder command +# Set up specific options to customise arduino-cli command def get_fqbn(b_name): if b_name in board_custom_fqbn and board_custom_fqbn[b_name]: return board_custom_fqbn[b_name] else: if b_name in board_options and board_options[b_name]: - return board_fqbn[b_name] + "," + board_options[b_name] + return f"{board_fqbn[b_name]},{board_options[b_name]}" else: return board_fqbn[b_name] -# Generate arduino builder basic command +# Generate arduino-cli basic command def genBasicCommand(b_name): cmd = [] cmd.append(arduino_cli) cmd.append("compile") # cmd.append("-warnings=all") cmd.append("--build-path") - cmd.append(os.path.join(build_output_dir, b_name)) + cmd.append(build_output_dir / b_name) cmd.append("--build-cache-path") cmd.append(build_output_cache_dir) if args.verbose: cmd.append("--verbose") - if version.parse(arduino_cli_version) <= version.parse("0.10.0"): - cmd.append("--output") - cmd.append(os.path.join(output_dir, b_name, bin_dir, "dummy_sketch")) - else: - cmd.append("--output-dir") - cmd.append(os.path.join(output_dir, b_name, bin_dir)) + cmd.append("--output-dir") + cmd.append(output_dir / b_name / bin_dir) cmd.append("--fqbn") cmd.append(get_fqbn(b_name)) cmd.append("dummy_sketch") @@ -729,63 +717,46 @@ def genBasicCommand(b_name): def create_build_conf_list(): build_conf_list = [] - idx = 1 - for b_name in board_fqbn: + for idx, b_name in enumerate(board_fqbn, start=1): build_conf_list.append( - ( + [ b_name, idx, - len(board_fqbn), - os.path.join(output_dir, b_name), + output_dir / b_name, genBasicCommand(b_name), - ) + ] ) - idx += 1 return build_conf_list def build_config(sketch, boardSkipped): global nb_build_skipped - build_conf_list = create_build_conf_list() - - for idx in reversed(range(len(build_conf_list))): - build_conf_list[idx][4][-1] = sketch - build_conf_list[idx][4][-4] = build_conf_list[idx][4][-4].replace( - "dummy_sketch", os.path.basename(sketch) - ) - if na_sketch_pattern: - if build_conf_list[idx][0] in na_sketch_pattern: - for pattern in na_sketch_pattern[build_conf_list[idx][0]]: - if re.search(pattern, sketch, re.IGNORECASE): - print( - (build_format_result).format( - "{}/{}".format( - build_conf_list[idx][1], build_conf_list[idx][2] - ), - build_conf_list[idx][0], - "\033[33mskipped\033[0m", - 0.00, - ) - ) - - boardSkipped.append(build_conf_list[idx][0]) - del build_conf_list[idx] - nb_build_skipped += 1 - break - else: - # get specific sketch options to append to the fqbn - for pattern in sketch_options: - print - if pattern in sketch_options: - if re.search(pattern, sketch, re.IGNORECASE): - if build_conf_list[idx][4][-2].count(":") == 3: - build_conf_list[idx][4][-2] += ( - "," + sketch_options[pattern] - ) - else: - build_conf_list[idx][4][-2] += ( - ":" + sketch_options[pattern] - ) + build_conf_list = [] + build_conf_list_tmp = create_build_conf_list() + + while len(build_conf_list_tmp): + build_conf = build_conf_list_tmp.pop(0) + build_conf[idx_cmd][-1] = sketch + b_name = build_conf[idx_b_name] + s_sketch = str(sketch) + build_conf[idx_build] = len(build_conf_list) + 1 + if b_name in na_sketch_pattern: + for pattern in na_sketch_pattern[b_name]: + if re.search(pattern, s_sketch, re.IGNORECASE): + boardSkipped.append(b_name) + nb_build_skipped += 1 + break + else: + # Get specific sketch options to append to the fqbn + for pattern in sketch_options: + if re.search(pattern, s_sketch, re.IGNORECASE): + if build_conf[idx_cmd][-2].count(":") == 3: + build_conf[idx_cmd][-2] += f",{sketch_options[pattern]}" + else: + build_conf[idx_cmd][-2] += f":{sketch_options[pattern]}" + build_conf_list.append(build_conf) + else: + build_conf_list.append(build_conf) return build_conf_list @@ -793,32 +764,35 @@ def build_config(sketch, boardSkipped): def build_all(): create_output_log_tree() wrapper = textwrap.TextWrapper(width=76) + if args.dry: + print("Performing a dry run (no build)") + build = dry_build + else: + build = real_build for sketch_nb, sketch in enumerate(sketch_list, start=1): boardKo = [] boardSkipped = [] - print("\n") - print(build_separator) + print(f"\n{build_separator}") print( "| {:^85} |".format( - "Sketch \033[34m{}\033[0m ({}/{})".format( - os.path.basename(sketch), sketch_nb, len(sketch_list) - ) + f"Sketch \033[34m{sketch.name}\033[0m ({sketch_nb}/{len(sketch_list)})" ) ) - wrapped_path_ = wrapper.wrap(text=os.path.dirname(sketch)) + wrapped_path_ = wrapper.wrap(text=str(sketch.parent)) for line in wrapped_path_: - print("| {:^76} |".format("{}".format(line))) + print("| {:^76} |".format(f"{line}")) print(build_separator) print((build_format_header).format("Num", "Board", "Result", "Time")) print(build_separator) build_conf_list = build_config(sketch, boardSkipped) + nb_build_conf = len(build_conf_list) with concurrent.futures.ProcessPoolExecutor(os.cpu_count() - 1) as executor: for build_conf, res in zip( build_conf_list, executor.map(build, build_conf_list) ): - check_status(res, build_conf, boardKo) + check_status(res, build_conf, boardKo, nb_build_conf) log_sketch_build_result(sketch, boardKo, boardSkipped) # Ensure no cache issue deleteFolder(build_output_cache_dir) @@ -826,12 +800,10 @@ def build_all(): # Run arduino-cli command -def build(build_conf): - cmd = build_conf[4] +def real_build(build_conf): + cmd = build_conf[idx_cmd] status = [time.monotonic()] - with open( - os.path.join(build_conf[3], os.path.basename(cmd[-1]) + ".log"), "w" - ) as stdout: + with open(build_conf[idx_log] / f"{cmd[-1].name}.log", "w") as stdout: res = subprocess.Popen(cmd, stdout=stdout, stderr=subprocess.STDOUT) res.wait() status[0] = time.monotonic() - status[0] @@ -839,6 +811,23 @@ def build(build_conf): return status +# Run arduino-cli command +def dry_build(build_conf): + # cmd = build_conf[4] + status = [random.random() * 10, random.randint(0, 1)] + if status[1] == 1: + # Create dummy log file + logFile = build_conf[idx_log] / f"{ build_conf[idx_cmd][-1].name}.log" + # random failed + dummy = open(logFile, "w") + if random.randint(0, 1) == 1: + dummy.writelines("region `FLASH' overflowed by 5612 bytes") + else: + dummy.writelines("Error:") + dummy.close() + return status + + # Parser parser = argparse.ArgumentParser( description="Manage arduino-cli to build sketche(s) for STM32 boards." @@ -867,29 +856,32 @@ def build(build_conf): parser.add_argument( "-c", "--clean", help="clean output directory.", action="store_true" ) + +parser.add_argument( + "-d", "--dry", help="perform a dry run (no build)", action="store_true" +) + parser.add_argument( "--arch", metavar="architecture", - help="core architecture to build. Default build architecture is '" - + arch_default - + "'", + help=f"core architecture to build. Default build architecture is {arch_default}", ) parser.add_argument( "--config", metavar="", - help="JSON file containing the build configuration for one or more\ + help=f"JSON file containing the build configuration for one or more\ maintainer/architecture. Board options for build, applicability of\ sketches for boards or required options. If sketch is not listed\ - then applicable to all board. Default core configuration is for '" - + arch_default - + " 'architecture in: " - + cores_config_file_default, + then applicable to all board. Default core configuration is for\ + '{arch_default}' architecture in: {cores_config_file_default}", ) parser.add_argument( - "-u", "--url", metavar="", help="additional URL for the board manager\ - Default url : " - + stm32_url, + "-u", + "--url", + metavar="", + help=f"additional URL for the board manager\ + Default url : {stm32_url}", ) parser.add_argument( @@ -901,7 +893,7 @@ def build(build_conf): # Sketch options sketchg0 = parser.add_argument_group( - title="Sketch(es) options", description="By default build " + sketch_default + title="Sketch(es) options", description=f"By default build {sketch_default}" ) sketchg1 = sketchg0.add_mutually_exclusive_group() @@ -924,9 +916,8 @@ def build(build_conf): "-e", "--exclude", metavar="", - help="file containing sketches pattern to ignore.\ - Default path : " - + exclude_file_default, + help=f"file containing sketches pattern to ignore.\ + Default path : {exclude_file_default}", ) args = parser.parse_args() @@ -939,18 +930,19 @@ def main(): deleteFolder(root_output_dir) load_core_config() - find_board() + if args.list != "sketch": + find_board() if args.list == "board": - print("%i board(s) available" % len(board_fqbn)) for board in board_fqbn: print(board) + print(f"{len(board_fqbn)} board(s) available") quit() manage_inos() if args.list == "sketch": for sketch in sketch_list: print(sketch) - print("%i sketches found" % len(sketch_list)) + print(f"{len(sketch_list)} sketches found") quit() if core_config: diff --git a/CI/build/conf/path_config_travis.json b/CI/build/conf/path_config_travis.json deleted file mode 100644 index 597663e91d..0000000000 --- a/CI/build/conf/path_config_travis.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "ARDUINO_PATH": "/home/travis/IDE/arduino", - "ARDUINO_PACKAGES": "/home/travis/.arduino15/packages", - "ARDUINO_USER_SKETCHBOOK": "/home/travis/Arduino", - "BUILD_OUPUT_DIR": "/tmp/BuildOutput", - "ROOT_OUPUT_DIR": "/home/travis/arduinoBuilderOutput" -} \ No newline at end of file diff --git a/CI/update/.flake8 b/CI/update/.flake8 new file mode 100644 index 0000000000..c55e852d7d --- /dev/null +++ b/CI/update/.flake8 @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 95 +ignore = W503, E203, E402 diff --git a/CI/utils/README.md b/CI/update/README.md similarity index 100% rename from CI/utils/README.md rename to CI/update/README.md diff --git a/CI/utils/fqbn.py b/CI/update/fqbn.py similarity index 96% rename from CI/utils/fqbn.py rename to CI/update/fqbn.py index 71493e213b..4c9dde4e5e 100644 --- a/CI/utils/fqbn.py +++ b/CI/update/fqbn.py @@ -22,7 +22,10 @@ "-b", "--board", metavar="pattern", help="pattern to find one or more board(s) fqbn" ) parser.add_argument( - "-p", "--path", metavar="", help="Path to the arduino-cli tool.", + "-p", + "--path", + metavar="", + help="Path to the arduino-cli tool.", ) args = parser.parse_args() @@ -91,9 +94,7 @@ def main(): arduino_cli_path = args.path assert os.path.exists( arduino_cli_path - ), "Path does not exist: '{}'. Please check the path!".format( - arduino_cli_path - ) + ), "Path does not exist: '{}'. Please check the path!".format(arduino_cli_path) if sys.platform.startswith("win32"): arduino_cli = os.path.join(arduino_cli_path, "arduino-cli.exe") else: diff --git a/CI/utils/gen_wrapper.sh b/CI/update/gen_wrapper.sh old mode 100755 new mode 100644 similarity index 100% rename from CI/utils/gen_wrapper.sh rename to CI/update/gen_wrapper.sh diff --git a/CI/utils/patch/HAL/G4/0001-G4-Fix-LL-FMC-build-issue-introduce-in-HAL-version-v.patch b/CI/update/patch/HAL/G4/0001-G4-Fix-LL-FMC-build-issue-introduce-in-HAL-version-v.patch similarity index 100% rename from CI/utils/patch/HAL/G4/0001-G4-Fix-LL-FMC-build-issue-introduce-in-HAL-version-v.patch rename to CI/update/patch/HAL/G4/0001-G4-Fix-LL-FMC-build-issue-introduce-in-HAL-version-v.patch diff --git a/CI/utils/stm32cube.py b/CI/update/stm32cube.py similarity index 99% rename from CI/utils/stm32cube.py rename to CI/update/stm32cube.py index 39c82b7b2a..6785084aa2 100644 --- a/CI/utils/stm32cube.py +++ b/CI/update/stm32cube.py @@ -9,15 +9,17 @@ from jinja2 import Environment, FileSystemLoader from packaging import version from pathlib import Path -from stm32common import createFolder, deleteFolder, copyFolder, copyFile, genSTM32List from urllib.parse import urljoin +script_path = Path(__file__).parent.resolve() +sys.path.append(str(script_path.parent)) +from utils import copyFile, copyFolder, createFolder, deleteFolder, genSTM32List + if sys.platform.startswith("win32"): from colorama import init init(autoreset=True) -script_path = Path(__file__).parent.resolve() home = Path.home() path_config_filename = "update_config.json" diff --git a/CI/utils/stm32variant.py b/CI/update/stm32variant.py similarity index 95% rename from CI/utils/stm32variant.py rename to CI/update/stm32variant.py index a30476011a..1bcfb6b702 100644 --- a/CI/utils/stm32variant.py +++ b/CI/update/stm32variant.py @@ -863,25 +863,25 @@ def qspi_pinmap(lst): name = "OCTOSPI" hal = "OSPI" if lst == quadspidata0_list: - aname = name + "_DATA0" + aname = f"{name}_DATA0" elif lst == quadspidata1_list: - aname = name + "_DATA1" + aname = f"{name}_DATA1" elif lst == quadspidata2_list: - aname = name + "_DATA2" + aname = f"{name}_DATA2" elif lst == quadspidata3_list: - aname = name + "_DATA3" + aname = f"{name}_DATA3" elif lst == quadspidata4_list: - aname = name + "_DATA4" + aname = f"{name}_DATA4" elif lst == quadspidata5_list: - aname = name + "_DATA5" + aname = f"{name}_DATA5" elif lst == quadspidata6_list: - aname = name + "_DATA6" + aname = f"{name}_DATA6" elif lst == quadspidata7_list: - aname = name + "_DATA7" + aname = f"{name}_DATA7" elif lst == quadspisclk_list: - aname = name + "_SCLK" + aname = f"{name}_SCLK" else: - aname = name + "_SSEL" + aname = f"{name}_SSEL" for p in lst: # 2nd element is the XXXXSPI_YYYY signal instm = re.match(ospi_regex, p[2]) @@ -1093,7 +1093,7 @@ def manage_syswkup(): num -= 1 cmt = "" else: - cmt = " /* " + p[2] + " */" + cmt = f" /* {p[2]} */" syswkup_pins_list[num].append([p[0], cmt]) return syswkup_pins_list @@ -1144,7 +1144,7 @@ def print_pinamevar(): for idx, syswkup_list in enumerate(syswkup_pins_list, start=1): if len(syswkup_list) > 1: for idx2, lst in enumerate(syswkup_list[1:], start=1): - alt_syswkup_list.append("{}_{}".format(idx, idx2)) + alt_syswkup_list.append(f"{idx}_{idx2}") return alt_syswkup_list @@ -1246,7 +1246,7 @@ def serial_pins_variant(): if serialnum: serialnum = serialnum.group(1) if serial_inst.startswith("LP"): - serialnum = "10" + serialnum + serialnum = f"10{serialnum}" else: print("No serial instance number found!") serialnum = "-1" @@ -1302,8 +1302,8 @@ def print_variant(generic_list, alt_syswkup_list): for idx, io in enumerate(io_list): pyn = io[0].replace("_", "", 1) if [item for item in adclist if item[0] == io[0]]: - ax = "A{}".format(analog_index) - pins_number_list.append({"name": pyn, "val": "PIN_" + ax}) + ax = f"A{analog_index}" + pins_number_list.append({"name": pyn, "val": f"PIN_{ax}"}) pinnames_list.append({"name": io[0], "ax": analog_index}) analog_pins_list.append({"val": idx, "ax": ax, "pyn": pyn}) analog_index += 1 @@ -1314,8 +1314,8 @@ def print_variant(generic_list, alt_syswkup_list): for idx, io in enumerate(dualpad_list): pyn = io[0].replace("_", "", 1) if [item for item in adclist if item[0] == io[0]]: - ax = "A{}".format(analog_index) - pins_number_list.append({"name": pyn, "val": "PIN_" + ax}) + ax = f"A{analog_index}" + pins_number_list.append({"name": pyn, "val": f"PIN_{ax}"}) pinnames_list.append({"name": io[0], "ax": analog_index}) analog_pins_list.append({"val": idx + idx_sum, "ax": ax, "pyn": pyn}) analog_index += 1 @@ -1326,8 +1326,8 @@ def print_variant(generic_list, alt_syswkup_list): for idx, io in enumerate(remap_list): pyn = io[0].replace("_", "", 1) if [item for item in adclist if item[0] == io[0]]: - ax = "A{}".format(analog_index) - pins_number_list.append({"name": pyn, "val": "PIN_" + ax}) + ax = f"A{analog_index}" + pins_number_list.append({"name": pyn, "val": f"PIN_{ax}"}) pinnames_list.append({"name": io[0], "ax": analog_index}) analog_pins_list.append({"val": idx + idx_sum, "ax": ax, "pyn": pyn}) analog_index += 1 @@ -1822,7 +1822,7 @@ def group_by_flash(group_base_list, glist, index_mcu_base): ] # Merge key if key: - packages_per_flash[key + "-" + flash] = packages_per_flash.pop(key) + packages_per_flash[f"{key}-{flash}"] = packages_per_flash.pop(key) else: packages_per_flash[flash] = packages_list @@ -1832,7 +1832,7 @@ def group_by_flash(group_base_list, glist, index_mcu_base): if len(key) == 1: new_mcu_dirname += key else: - new_mcu_dirname += "(" + key + ")" + new_mcu_dirname += f"({key})" # Handle package with ANPQX # One case not manage: [Tx, TxX, Yx] # Assuming it is not an issue to have non existing mcu @@ -1845,9 +1845,7 @@ def group_by_flash(group_base_list, glist, index_mcu_base): # Assert if sub.group(2) != "x": print( - "Package of {} info contains {} instead of 'x'".format( - base_name, sub.group(2) - ) + "Package of {base_name} info contains {sub.group(2)} instead of 'x'" ) exit(1) if sub.group(3): @@ -1858,7 +1856,7 @@ def group_by_flash(group_base_list, glist, index_mcu_base): if len(pcounter) == 1: new_mcu_dirname += package_list[0] else: - new_mcu_dirname += "(" + "-".join(k for k in sorted(pcounter)) + ")" + new_mcu_dirname += f"({'-'.join(k for k in sorted(pcounter))})" if len(ecounter): new_mcu_dirname += "x" if (len(ecounter) == 1) and ( @@ -1867,7 +1865,7 @@ def group_by_flash(group_base_list, glist, index_mcu_base): # new_mcu_dirname += next(iter(ecounter)) new_mcu_dirname += ext_list[0] else: - new_mcu_dirname += "(" + "-".join(k for k in sorted(ecounter)) + ")" + new_mcu_dirname += f"({'-'.join(k for k in sorted(ecounter))})" del package_list[:] del ext_list[:] @@ -1903,12 +1901,11 @@ def merge_dir(out_temp_path, group_mcu_dir, mcu_family, periph_xml, variant_exp) for index, glist in enumerate(group_base_list): # Only one mcu if len(glist) == 1: - new_mcu_dirname += ("_" if index != 0 else "") + glist[0].strip("x") + new_mcu_dirname += f"{'_' if index != 0 else ''}{glist[0].strip('x')}" else: # Group using flash info - new_mcu_dirname += ("_" if index != 0 else "") + group_by_flash( - group_base_list, glist, index_mcu_base - ) + gbf = group_by_flash(group_base_list, glist, index_mcu_base) + new_mcu_dirname += f"{'_' if index != 0 else ''}{gbf}" del group_package_list[:] del group_flash_list[:] del group_base_list[:] @@ -1951,23 +1948,23 @@ def merge_dir(out_temp_path, group_mcu_dir, mcu_family, periph_xml, variant_exp) new_line_c = periph_xml.pop(0) for index, xml in enumerate(periph_xml, 1): if index % 2 == 0: - new_line_c += "\n * {}".format(xml) + new_line_c += f"\n * {xml}" else: - new_line_c += ", {}".format(xml) + new_line_c += f", {xml}" update_file(mcu_dir / periph_c_filename, periperalpins_regex, new_line_c) variant_exp.sort() variant_exp = list(OrderedDict.fromkeys(variant_exp)) new_line_c = variant_exp[0] - new_line_h = "{}".format(variant_exp.pop(0)) + new_line_h = f"{variant_exp.pop(0)}" for index, pre in enumerate(variant_exp, 1): if index % 2 == 0: - new_line_c += " ||\\\n {}".format(pre) - new_line_h += " &&\\\n !{}".format(pre) + new_line_c += f" ||\\\n {pre}" + new_line_h += f" &&\\\n !{pre}" else: - new_line_c += " || {}".format(pre) - new_line_h += " && !{}".format(pre) + new_line_c += f" || {pre}" + new_line_h += f" && !{pre}" update_file(mcu_dir / variant_cpp_filename, update_regex, new_line_c) update_file(mcu_dir / generic_clock_filename, update_regex, new_line_c) update_file(mcu_dir / variant_h_filename, update_regex, new_line_h) @@ -2102,7 +2099,7 @@ def default_cubemxdir(): def create_config(): # Create a Json file for a better path management try: - print("Please set your configuration in '{}' file".format(config_filename)) + print(f"Please set your configuration in '{config_filename}' file") config_file = open(config_filename, "w", newline="\n") config_file.write( json.dumps( @@ -2115,7 +2112,7 @@ def create_config(): ) config_file.close() except IOError: - print("Failed to open " + config_filename) + print(f"Failed to open {config_filename}") exit(1) @@ -2141,7 +2138,7 @@ def check_config(): if conf: cubemxdir = Path(conf) except IOError: - print("Failed to open " + config_filename) + print(f"Failed to open {config_filename}") else: create_config() @@ -2152,7 +2149,7 @@ def manage_repo(): try: if not args.skip: - print("Updating " + repo_name + "...") + print(f"Updating {repo_name}...") if repo_path.is_dir(): # Get new tags from the remote git_cmds = [ @@ -2189,7 +2186,7 @@ def manage_repo(): db_release = version_tag return True except subprocess.CalledProcessError as e: - print("Command {} failed with error code {}".format(e.cmd, e.returncode)) + print(f"Command {e.cmd} failed with error code {e.returncode}") return False @@ -2221,32 +2218,22 @@ def manage_repo(): # By default, generate for all mcu xml files description parser = argparse.ArgumentParser( description=textwrap.dedent( - """\ + f""" By default, generates: - - {}, - - {}, - - {}, - - {}, - - {} - - {} + - {periph_c_filename}, + - {pinvar_h_filename}, + - {variant_cpp_filename}, + - {variant_h_filename}, + - {boards_entry_filename} + - {generic_clock_filename} for all xml files description available in STM32CubeMX internal database. -Internal database path must be defined in {}. +Internal database path must be defined in {config_filename}. It can be the one from STM32CubeMX directory if defined: -\t{} +\t{cubemxdir} or the one from GitHub: -\t{} - -""".format( - periph_c_filename, - pinvar_h_filename, - variant_cpp_filename, - variant_h_filename, - boards_entry_filename, - generic_clock_filename, - config_filename, - cubemxdir, - gh_url, - ) +\t{gh_url} + +""" ), epilog=textwrap.dedent( """\ @@ -2287,18 +2274,16 @@ def manage_repo(): "-c", "--cube", help=textwrap.dedent( - """\ -Use STM32CubeMX internal database. Default use GitHub {} repository. -""".format( - repo_name - ) + f"""\ +Use STM32CubeMX internal database. Default use GitHub {repo_name} repository. +""" ), action="store_true", ) parser.add_argument( "-s", "--skip", - help="Skip {} clone/fetch".format(repo_name), + help=f"Skip {repo_name} clone/fetch", action="store_true", ) args = parser.parse_args() @@ -2315,12 +2300,10 @@ def manage_repo(): if fallback or args.cube: if not (cubemxdir.is_dir()): print( - """ + f""" Cube Mx seems not to be installed or not at the specified location. -Please check the value set for 'CUBEMX_DIRECTORY' in '{}' file.""".format( - config_filename - ) +Please check the value set for 'CUBEMX_DIRECTORY' in '{config_filename}' file.""" ) quit() @@ -2339,7 +2322,7 @@ def manage_repo(): release_match = re.match(release_regex, db_release) if release_match: db_release = release_match.group(1) -print("CubeMX DB release {}\n".format(db_release)) +print(f"CubeMX DB release {db_release}\n") # if args.mcu: # # Check input file exists @@ -2382,11 +2365,11 @@ def manage_repo(): xml_mcu.unlink() continue - print("Generating files for '{}'...".format(mcu_file.name)) + print(f"Generating files for '{mcu_file.name}'...") if not gpiofile: print("Could not find GPIO file") quit() - xml_gpio = parse(str(dirIP / ("GPIO-" + gpiofile + "_Modes.xml"))) + xml_gpio = parse(str(dirIP / f"GPIO-{gpiofile}_Modes.xml")) mcu_family_dir = mcu_family + "xx" out_temp_path = tmp_dir / mcu_family_dir / mcu_file.stem.replace("STM32", "") @@ -2415,17 +2398,14 @@ def manage_repo(): print_variant(generic_list, alt_syswkup_list) del alt_syswkup_list[:] del generic_list[:] - print( - "* Total I/O pins found: {}".format( - len(io_list) + len(alt_list) + len(dualpad_list) + len(remap_list) - ) - ) - print(" - {} I/O pins".format(len(io_list))) + sum_io = len(io_list) + len(alt_list) + len(dualpad_list) + len(remap_list) + print(f"* Total I/O pins found: {sum_io}") + print(f" - {len(io_list)} I/O pins") if len(dualpad_list): - print(" - {} dual pad".format(len(dualpad_list))) + print(f" - {len(dualpad_list)} dual pad") if len(remap_list): - print(" - {} remap pins".format(len(remap_list))) - print(" - {} ALT I/O pins".format(len(alt_list))) + print(f" - {len(remap_list)} remap pins") + print(f" - {len(alt_list)} ALT I/O pins") # for io in io_list: # print(io[0] + ", " + io[1]) diff --git a/CI/utils/stm32wrapper.py b/CI/update/stm32wrapper.py similarity index 91% rename from CI/utils/stm32wrapper.py rename to CI/update/stm32wrapper.py index 5830405443..04d90a9e06 100644 --- a/CI/utils/stm32wrapper.py +++ b/CI/update/stm32wrapper.py @@ -1,11 +1,14 @@ import argparse import re +import sys from itertools import groupby from jinja2 import Environment, FileSystemLoader, Template from pathlib import Path -from stm32common import createFolder, deleteFolder, genSTM32List script_path = Path(__file__).parent.resolve() +sys.path.append(str(script_path.parent)) +from utils import createFolder, deleteFolder, genSTM32List + # Base path core_path = script_path.parent.parent SrcWrapper_path = "" @@ -80,7 +83,7 @@ def checkConfig(arg_core, arg_cmsis): CMSIS_path = core_path.parent / "ArduinoModule-CMSIS" / "CMSIS_5" if not core_path.is_dir(): - print("Could not find " + core_path) + print(f"Could not find {core_path}") exit(1) system_path = core_path / "system" @@ -106,7 +109,7 @@ def printCMSISStartup(log): filelist = [pth.name for pth in filelist] if len(filelist): if log: - print("Number of startup files: {}".format(len(filelist))) + print(f"Number of startup files: {len(filelist)}") # Some mcu have two startup files # Ex: WL one for cm0plus and one for cm4 # In that case this is the same value line so add an extra defined @@ -138,7 +141,7 @@ def printSystemSTM32(log): filelist = sorted(system_path.glob("STM32*/system_stm32*.c")) if len(filelist): if log: - print("Number of system stm32 files: {}".format(len(filelist))) + print(f"Number of system stm32 files: {len(filelist)}") system_list = [] for fp in filelist: system_list.append({"serie": fp.parent.name, "fn": fp.name}) @@ -172,16 +175,16 @@ def wrap(arg_core, arg_cmsis, log): hal_c_dict = {} # Search all files for each series for serie in stm32_series: - src = HALDrivers_path / ("STM32" + serie + "xx_HAL_Driver") / "Src" - inc = HALDrivers_path / ("STM32" + serie + "xx_HAL_Driver") / "Inc" + src = HALDrivers_path / f"STM32{serie}xx_HAL_Driver" / "Src" + inc = HALDrivers_path / f"STM32{serie}xx_HAL_Driver" / "Inc" if src.exists(): if log: - print("Generating for " + serie + "...") + print(f"Generating for {serie}...") lower = serie.lower() # Search stm32yyxx_[hal|ll]*.c file - filelist = src.glob("stm32" + lower + "xx_*.c") + filelist = src.glob(f"stm32{lower}xx_*.c") for fp in filelist: # File name fn = fp.name @@ -201,7 +204,7 @@ def wrap(arg_core, arg_cmsis, log): hal_c_dict[peripheral] = [lower] # Search stm32yyxx_ll_*.h file - filelist = inc.glob("stm32" + lower + "xx_ll_*.h") + filelist = inc.glob(f"stm32{lower}xx_ll_*.h") for fp in filelist: # File name fn = fp.name @@ -264,7 +267,7 @@ def wrap(arg_core, arg_cmsis, log): # CMSIS DSP C source file if not CMSIS_path.is_dir(): - print("Could not find {}".format(CMSIS_path)) + print(f"Could not find {CMSIS_path}") print("CMSIS DSP generation skipped.") else: # Delete all subfolders @@ -279,7 +282,7 @@ def wrap(arg_core, arg_cmsis, log): fdn = CMSIS_DSP_outSrc_path / dn if not fdn.is_dir(): createFolder(fdn) - out_file = open(fdn / (dn + ".c"), "w", newline="\n") + out_file = open(fdn / (f"{dn}.c"), "w", newline="\n") all_ll_file.write(dsp_file_template.render(dsp_path=dn)) out_file.close() return 0 @@ -288,19 +291,19 @@ def wrap(arg_core, arg_cmsis, log): if __name__ == "__main__": # Parser wrapparser = argparse.ArgumentParser( - description="Generate all wrappers files need by the STM32 core (HAL, LL, CMSIS, ...)" + description="Generate all wrappers files (HAL, LL, CMSIS, ...)" ) wrapparser.add_argument( "-c", "--core", metavar="core_path", - help="Root path of the STM32 core. Default: {}".format(core_path), + help=f"Root path of the STM32 core. Default: {core_path}", ) wrapparser.add_argument( "-s", "--cmsis", metavar="cmsis_path", - help="Root path of the CMSIS. Default: {}".format(CMSIS_path), + help=f"Root path of the CMSIS. Default: {CMSIS_path}", ) wrapargs = wrapparser.parse_args() diff --git a/CI/utils/templates/PeripheralPins.c b/CI/update/templates/PeripheralPins.c similarity index 100% rename from CI/utils/templates/PeripheralPins.c rename to CI/update/templates/PeripheralPins.c diff --git a/CI/utils/templates/PinNamesVar.h b/CI/update/templates/PinNamesVar.h similarity index 100% rename from CI/utils/templates/PinNamesVar.h rename to CI/update/templates/PinNamesVar.h diff --git a/CI/utils/templates/boards_entry.txt b/CI/update/templates/boards_entry.txt similarity index 100% rename from CI/utils/templates/boards_entry.txt rename to CI/update/templates/boards_entry.txt diff --git a/CI/utils/templates/generic_clock.c b/CI/update/templates/generic_clock.c similarity index 100% rename from CI/utils/templates/generic_clock.c rename to CI/update/templates/generic_clock.c diff --git a/CI/utils/templates/stm32_def_build.h b/CI/update/templates/stm32_def_build.h similarity index 100% rename from CI/utils/templates/stm32_def_build.h rename to CI/update/templates/stm32_def_build.h diff --git a/CI/utils/templates/stm32yyxx_hal_conf.h b/CI/update/templates/stm32yyxx_hal_conf.h similarity index 100% rename from CI/utils/templates/stm32yyxx_hal_conf.h rename to CI/update/templates/stm32yyxx_hal_conf.h diff --git a/CI/utils/templates/stm32yyxx_ll.h b/CI/update/templates/stm32yyxx_ll.h similarity index 100% rename from CI/utils/templates/stm32yyxx_ll.h rename to CI/update/templates/stm32yyxx_ll.h diff --git a/CI/utils/templates/stm32yyxx_ll_ppp.h b/CI/update/templates/stm32yyxx_ll_ppp.h similarity index 100% rename from CI/utils/templates/stm32yyxx_ll_ppp.h rename to CI/update/templates/stm32yyxx_ll_ppp.h diff --git a/CI/utils/templates/stm32yyxx_zz_ppp.c b/CI/update/templates/stm32yyxx_zz_ppp.c similarity index 100% rename from CI/utils/templates/stm32yyxx_zz_ppp.c rename to CI/update/templates/stm32yyxx_zz_ppp.c diff --git a/CI/utils/templates/system_stm32yyxx.c b/CI/update/templates/system_stm32yyxx.c similarity index 100% rename from CI/utils/templates/system_stm32yyxx.c rename to CI/update/templates/system_stm32yyxx.c diff --git a/CI/utils/templates/variant_generic.cpp b/CI/update/templates/variant_generic.cpp similarity index 100% rename from CI/utils/templates/variant_generic.cpp rename to CI/update/templates/variant_generic.cpp diff --git a/CI/utils/templates/variant_generic.h b/CI/update/templates/variant_generic.h similarity index 100% rename from CI/utils/templates/variant_generic.h rename to CI/update/templates/variant_generic.h diff --git a/CI/utils/__init__.py b/CI/utils/__init__.py new file mode 100644 index 0000000000..2f30247937 --- /dev/null +++ b/CI/utils/__init__.py @@ -0,0 +1 @@ +from .pathlib_ext import * diff --git a/CI/utils/stm32common.py b/CI/utils/pathlib_ext.py similarity index 77% rename from CI/utils/stm32common.py rename to CI/utils/pathlib_ext.py index f1aca9a163..27134f2624 100644 --- a/CI/utils/stm32common.py +++ b/CI/utils/pathlib_ext.py @@ -1,14 +1,14 @@ import re import shutil +import sys # Create a folder if not exists def createFolder(path): try: - if not path.exists(): - path.mkdir() + path.mkdir(parents=True, exist_ok=True) except OSError: - print("Error: Creating directory {}".format(path)) + print(f"Error: Creating directory {path}") # Delete targeted folder recursively @@ -23,7 +23,7 @@ def copyFolder(src, dest, ign_patt=set()): if src.is_dir(): shutil.copytree(src, dest, ignore=shutil.ignore_patterns(*ign_patt)) except OSError as e: - print("Error: Folder {} not copied. {}".format(src, e)) + print(f"Error: Folder {src} not copied. {e}") # copy one file to dest @@ -32,7 +32,7 @@ def copyFile(src, dest): if src.is_file(): shutil.copy(str(src), str(dest)) except OSError as e: - print("Error: File {} not copied. {}".format(src, e)) + print(f"Error: File {src} not copied. {e}") def genSTM32List(path, pattern): @@ -50,3 +50,8 @@ def genSTM32List(path, pattern): stm32_list.append(res.group(1)) stm32_list.sort() return stm32_list + + +if __name__ == "__main__": + print("This script is not intend to be called directly") + sys.exit()