From b469182d4af431b039361244d6f64c4bde18609e Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Thu, 7 Oct 2021 14:30:15 +0200 Subject: [PATCH 01/17] refactor(scripts): common python functions moved as a module Signed-off-by: Frederic Pillon --- CI/update/.flake8 | 3 +++ CI/{utils => update}/README.md | 0 CI/{utils => update}/fqbn.py | 9 +++++---- CI/{utils => update}/gen_wrapper.sh | 0 ...x-LL-FMC-build-issue-introduce-in-HAL-version-v.patch | 0 CI/{utils => update}/stm32cube.py | 6 ++++-- CI/{utils => update}/stm32variant.py | 0 CI/{utils => update}/stm32wrapper.py | 5 ++++- CI/{utils => update}/templates/PeripheralPins.c | 0 CI/{utils => update}/templates/PinNamesVar.h | 0 CI/{utils => update}/templates/boards_entry.txt | 0 CI/{utils => update}/templates/generic_clock.c | 0 CI/{utils => update}/templates/stm32_def_build.h | 0 CI/{utils => update}/templates/stm32yyxx_hal_conf.h | 0 CI/{utils => update}/templates/stm32yyxx_ll.h | 0 CI/{utils => update}/templates/stm32yyxx_ll_ppp.h | 0 CI/{utils => update}/templates/stm32yyxx_zz_ppp.c | 0 CI/{utils => update}/templates/system_stm32yyxx.c | 0 CI/{utils => update}/templates/variant_generic.cpp | 0 CI/{utils => update}/templates/variant_generic.h | 0 CI/utils/__init__.py | 1 + CI/utils/{stm32common.py => pathlib_ext.py} | 5 +++++ 22 files changed, 22 insertions(+), 7 deletions(-) create mode 100644 CI/update/.flake8 rename CI/{utils => update}/README.md (100%) rename CI/{utils => update}/fqbn.py (96%) rename CI/{utils => update}/gen_wrapper.sh (100%) mode change 100755 => 100644 rename CI/{utils => update}/patch/HAL/G4/0001-G4-Fix-LL-FMC-build-issue-introduce-in-HAL-version-v.patch (100%) rename CI/{utils => update}/stm32cube.py (99%) rename CI/{utils => update}/stm32variant.py (100%) rename CI/{utils => update}/stm32wrapper.py (98%) rename CI/{utils => update}/templates/PeripheralPins.c (100%) rename CI/{utils => update}/templates/PinNamesVar.h (100%) rename CI/{utils => update}/templates/boards_entry.txt (100%) rename CI/{utils => update}/templates/generic_clock.c (100%) rename CI/{utils => update}/templates/stm32_def_build.h (100%) rename CI/{utils => update}/templates/stm32yyxx_hal_conf.h (100%) rename CI/{utils => update}/templates/stm32yyxx_ll.h (100%) rename CI/{utils => update}/templates/stm32yyxx_ll_ppp.h (100%) rename CI/{utils => update}/templates/stm32yyxx_zz_ppp.c (100%) rename CI/{utils => update}/templates/system_stm32yyxx.c (100%) rename CI/{utils => update}/templates/variant_generic.cpp (100%) rename CI/{utils => update}/templates/variant_generic.h (100%) create mode 100644 CI/utils/__init__.py rename CI/utils/{stm32common.py => pathlib_ext.py} (92%) diff --git a/CI/update/.flake8 b/CI/update/.flake8 new file mode 100644 index 0000000000..d74cc78fd4 --- /dev/null +++ b/CI/update/.flake8 @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 95 +ignore = W503, E203 diff --git a/CI/utils/README.md b/CI/update/README.md similarity index 100% rename from CI/utils/README.md rename to CI/update/README.md diff --git a/CI/utils/fqbn.py b/CI/update/fqbn.py similarity index 96% rename from CI/utils/fqbn.py rename to CI/update/fqbn.py index 71493e213b..4c9dde4e5e 100644 --- a/CI/utils/fqbn.py +++ b/CI/update/fqbn.py @@ -22,7 +22,10 @@ "-b", "--board", metavar="pattern", help="pattern to find one or more board(s) fqbn" ) parser.add_argument( - "-p", "--path", metavar="", help="Path to the arduino-cli tool.", + "-p", + "--path", + metavar="", + help="Path to the arduino-cli tool.", ) args = parser.parse_args() @@ -91,9 +94,7 @@ def main(): arduino_cli_path = args.path assert os.path.exists( arduino_cli_path - ), "Path does not exist: '{}'. Please check the path!".format( - arduino_cli_path - ) + ), "Path does not exist: '{}'. Please check the path!".format(arduino_cli_path) if sys.platform.startswith("win32"): arduino_cli = os.path.join(arduino_cli_path, "arduino-cli.exe") else: diff --git a/CI/utils/gen_wrapper.sh b/CI/update/gen_wrapper.sh old mode 100755 new mode 100644 similarity index 100% rename from CI/utils/gen_wrapper.sh rename to CI/update/gen_wrapper.sh diff --git a/CI/utils/patch/HAL/G4/0001-G4-Fix-LL-FMC-build-issue-introduce-in-HAL-version-v.patch b/CI/update/patch/HAL/G4/0001-G4-Fix-LL-FMC-build-issue-introduce-in-HAL-version-v.patch similarity index 100% rename from CI/utils/patch/HAL/G4/0001-G4-Fix-LL-FMC-build-issue-introduce-in-HAL-version-v.patch rename to CI/update/patch/HAL/G4/0001-G4-Fix-LL-FMC-build-issue-introduce-in-HAL-version-v.patch diff --git a/CI/utils/stm32cube.py b/CI/update/stm32cube.py similarity index 99% rename from CI/utils/stm32cube.py rename to CI/update/stm32cube.py index 39c82b7b2a..6785084aa2 100644 --- a/CI/utils/stm32cube.py +++ b/CI/update/stm32cube.py @@ -9,15 +9,17 @@ from jinja2 import Environment, FileSystemLoader from packaging import version from pathlib import Path -from stm32common import createFolder, deleteFolder, copyFolder, copyFile, genSTM32List from urllib.parse import urljoin +script_path = Path(__file__).parent.resolve() +sys.path.append(str(script_path.parent)) +from utils import copyFile, copyFolder, createFolder, deleteFolder, genSTM32List + if sys.platform.startswith("win32"): from colorama import init init(autoreset=True) -script_path = Path(__file__).parent.resolve() home = Path.home() path_config_filename = "update_config.json" diff --git a/CI/utils/stm32variant.py b/CI/update/stm32variant.py similarity index 100% rename from CI/utils/stm32variant.py rename to CI/update/stm32variant.py diff --git a/CI/utils/stm32wrapper.py b/CI/update/stm32wrapper.py similarity index 98% rename from CI/utils/stm32wrapper.py rename to CI/update/stm32wrapper.py index 5830405443..d0278e14de 100644 --- a/CI/utils/stm32wrapper.py +++ b/CI/update/stm32wrapper.py @@ -1,11 +1,14 @@ import argparse import re +import sys from itertools import groupby from jinja2 import Environment, FileSystemLoader, Template from pathlib import Path -from stm32common import createFolder, deleteFolder, genSTM32List script_path = Path(__file__).parent.resolve() +sys.path.append(str(script_path.parent)) +from utils import createFolder, deleteFolder, genSTM32List + # Base path core_path = script_path.parent.parent SrcWrapper_path = "" diff --git a/CI/utils/templates/PeripheralPins.c b/CI/update/templates/PeripheralPins.c similarity index 100% rename from CI/utils/templates/PeripheralPins.c rename to CI/update/templates/PeripheralPins.c diff --git a/CI/utils/templates/PinNamesVar.h b/CI/update/templates/PinNamesVar.h similarity index 100% rename from CI/utils/templates/PinNamesVar.h rename to CI/update/templates/PinNamesVar.h diff --git a/CI/utils/templates/boards_entry.txt b/CI/update/templates/boards_entry.txt similarity index 100% rename from CI/utils/templates/boards_entry.txt rename to CI/update/templates/boards_entry.txt diff --git a/CI/utils/templates/generic_clock.c b/CI/update/templates/generic_clock.c similarity index 100% rename from CI/utils/templates/generic_clock.c rename to CI/update/templates/generic_clock.c diff --git a/CI/utils/templates/stm32_def_build.h b/CI/update/templates/stm32_def_build.h similarity index 100% rename from CI/utils/templates/stm32_def_build.h rename to CI/update/templates/stm32_def_build.h diff --git a/CI/utils/templates/stm32yyxx_hal_conf.h b/CI/update/templates/stm32yyxx_hal_conf.h similarity index 100% rename from CI/utils/templates/stm32yyxx_hal_conf.h rename to CI/update/templates/stm32yyxx_hal_conf.h diff --git a/CI/utils/templates/stm32yyxx_ll.h b/CI/update/templates/stm32yyxx_ll.h similarity index 100% rename from CI/utils/templates/stm32yyxx_ll.h rename to CI/update/templates/stm32yyxx_ll.h diff --git a/CI/utils/templates/stm32yyxx_ll_ppp.h b/CI/update/templates/stm32yyxx_ll_ppp.h similarity index 100% rename from CI/utils/templates/stm32yyxx_ll_ppp.h rename to CI/update/templates/stm32yyxx_ll_ppp.h diff --git a/CI/utils/templates/stm32yyxx_zz_ppp.c b/CI/update/templates/stm32yyxx_zz_ppp.c similarity index 100% rename from CI/utils/templates/stm32yyxx_zz_ppp.c rename to CI/update/templates/stm32yyxx_zz_ppp.c diff --git a/CI/utils/templates/system_stm32yyxx.c b/CI/update/templates/system_stm32yyxx.c similarity index 100% rename from CI/utils/templates/system_stm32yyxx.c rename to CI/update/templates/system_stm32yyxx.c diff --git a/CI/utils/templates/variant_generic.cpp b/CI/update/templates/variant_generic.cpp similarity index 100% rename from CI/utils/templates/variant_generic.cpp rename to CI/update/templates/variant_generic.cpp diff --git a/CI/utils/templates/variant_generic.h b/CI/update/templates/variant_generic.h similarity index 100% rename from CI/utils/templates/variant_generic.h rename to CI/update/templates/variant_generic.h diff --git a/CI/utils/__init__.py b/CI/utils/__init__.py new file mode 100644 index 0000000000..2f30247937 --- /dev/null +++ b/CI/utils/__init__.py @@ -0,0 +1 @@ +from .pathlib_ext import * diff --git a/CI/utils/stm32common.py b/CI/utils/pathlib_ext.py similarity index 92% rename from CI/utils/stm32common.py rename to CI/utils/pathlib_ext.py index f1aca9a163..de08bd8239 100644 --- a/CI/utils/stm32common.py +++ b/CI/utils/pathlib_ext.py @@ -50,3 +50,8 @@ def genSTM32List(path, pattern): stm32_list.append(res.group(1)) stm32_list.sort() return stm32_list + + +if __name__ == "__main__": + print("This script is not intend to be called directly") + sys.exit() From 998ca309bc8b32d44e0b771af49563798cd43cce Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Fri, 8 Oct 2021 18:51:59 +0200 Subject: [PATCH 02/17] refactor(arduino_cli): update to use pathlib Signed-off-by: Frederic Pillon --- CI/build/arduino-cli.py | 248 +++++++++++++++++++--------------------- CI/utils/pathlib_ext.py | 3 +- 2 files changed, 119 insertions(+), 132 deletions(-) diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index f13ac2651e..8afc742f37 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -5,44 +5,47 @@ import json import os from packaging import version +from pathlib import Path import re -import shutil import subprocess import sys import tempfile import textwrap import time +script_path = Path(__file__).parent.resolve() +sys.path.append(str(script_path.parent)) +from utils import createFolder, deleteFolder + if sys.platform.startswith("win32"): from colorama import init init(autoreset=True) -home = os.path.expanduser("~") -tempdir = tempfile.gettempdir() -build_id = time.strftime("_%Y-%m-%d_%H-%M-%S") -script_path = os.path.dirname(os.path.abspath(__file__)) -path_config_filename = os.path.join(script_path, "path_config.json") +home = Path.home() +tempdir = Path(tempfile.gettempdir()) +build_id = time.strftime(".%Y-%m-%d_%H-%M-%S") +path_config_filename = script_path / "path_config.json" -arduino_cli_path = "" +arduino_cli_path = Path("") stm32_url = "https://github.com/stm32duino/BoardManagerFiles/raw/main/package_stmicroelectronics_index.json" sketches_path_list = [] -default_build_output_dir = os.path.join(tempdir, "build_arduinoCliOutput") -build_output_dir = os.path.join(tempdir, "build_arduinoCliOutput" + build_id) -build_output_cache_dir = os.path.join(build_output_dir, "cache") -root_output_dir = os.path.join(home, "Documents", "arduinoCliOutput") +default_build_output_dir = tempdir / "build_arduinoCliOutput" +build_output_dir = tempdir / f"build_arduinoCliOutput{build_id}" +build_output_cache_dir = build_output_dir / "cache" +root_output_dir = home / "Documents" / "arduinoCliOutput" -output_dir = "" -log_file = "" +output_dir = Path("") +log_file = Path("") -# Ouput directory path +# Output directory path bin_dir = "binaries" # Default -sketch_default = os.path.join(script_path, "examples", "BareMinimum") -exclude_file_default = os.path.join(script_path, "conf", "exclude_list.txt") -cores_config_file_default = os.path.join(script_path, "conf", "cores_config.json") -cores_config_file_ci = os.path.join(script_path, "conf", "cores_config_ci.json") +sketch_default = script_path / "examples" / "BareMinimum" +exclude_file_default = script_path / "conf" / "exclude_list.txt" +cores_config_file_default = script_path / "conf" / "cores_config.json" +cores_config_file_ci = script_path / "conf" / "cores_config_ci.json" maintainer_default = "STMicroelectronics" arch_default = "stm32" @@ -87,21 +90,6 @@ build_separator = "-" * 80 -# Create a folder if not exists -def createFolder(folder): - try: - if not os.path.exists(folder): - os.makedirs(folder) - except OSError: - print("Error: Creating directory. " + folder) - - -# Delete targeted folder recursively -def deleteFolder(folder): - if os.path.isdir(folder): - shutil.rmtree(folder, ignore_errors=True) - - def cat(file): with open(file, "r") as f: print(f.read()) @@ -114,15 +102,13 @@ def create_output_log_tree(): with open(log_file, "w") as file: file.write(build_separator + "\nStarts ") file.write(time.strftime("%A %d %B %Y %H:%M:%S ")) - file.write( - "\nLog will be available here:\n{}\n".format(os.path.abspath(output_dir)) - ) file.write(build_separator + "\n") + file.write(f"\nLog will be available here:\n{output_dir.resolve()}\n") # Folders for board in board_fqbn: - createFolder(os.path.join(output_dir, board, bin_dir)) - createFolder(os.path.join(output_dir, board)) - createFolder(os.path.join(build_output_dir, board)) + createFolder(output_dir / board / bin_dir) + createFolder(output_dir / board) + createFolder(build_output_dir / board) def create_config(): @@ -140,10 +126,10 @@ def create_config(): path_config_file.write( json.dumps( { - "ARDUINO_CLI_PATH": arduino_cli_path, - "SKETCHES_PATHS": sketches_path_list, - "BUILD_OUPUT_DIR": default_build_output_dir, - "ROOT_OUPUT_DIR": root_output_dir, + "ARDUINO_CLI_PATH": str(arduino_cli_path), + "SKETCHES_PATHS": [str(fp) for fp in sketches_path_list], + "BUILD_OUPUT_DIR": str(default_build_output_dir), + "ROOT_OUPUT_DIR": str(root_output_dir), }, indent=2, ) @@ -164,43 +150,40 @@ def check_config(): global stm32_url if args.ci is False: - if os.path.isfile(path_config_filename): + if path_config_filename.is_file(): try: path_config_file = open(path_config_filename, "r") path_config = json.load(path_config_file) # Common path - arduino_cli_path = path_config["ARDUINO_CLI_PATH"] - sketches_path_list = path_config["SKETCHES_PATHS"] - build_output_dir = path_config["BUILD_OUPUT_DIR"] + build_id - root_output_dir = path_config["ROOT_OUPUT_DIR"] + arduino_cli_path = Path(path_config["ARDUINO_CLI_PATH"]) + sketches_path_list = [Path(fp) for fp in path_config["SKETCHES_PATHS"]] + build_output_dir = Path(path_config["BUILD_OUPUT_DIR"]) + build_output_dir = build_output_dir.with_suffix(build_id) + root_output_dir = Path(path_config["ROOT_OUPUT_DIR"]) path_config_file.close() except IOError: print("Failed to open " + path_config_file) else: create_config() - output_dir = os.path.join(root_output_dir, "build" + build_id) - log_file = os.path.join(output_dir, "build_result.log") + output_dir = root_output_dir / f"build{build_id}" + log_file = output_dir / "build_result.log" - if arduino_cli_path != "": - assert os.path.exists( - arduino_cli_path - ), "Path does not exist: {} . Please check the path in the json config file".format( - arduino_cli_path + if arduino_cli_path != Path(): + assert arduino_cli_path.exists(), ( + f"Path {arduino_cli_path} does not exist." + f" Please check path in the json config file." ) if sys.platform.startswith("win32"): - arduino_cli = os.path.join(arduino_cli_path, "arduino-cli.exe") - elif sys.platform.startswith("linux"): - arduino_cli = os.path.join(arduino_cli_path, "arduino-cli") - elif sys.platform.startswith("darwin"): - arduino_cli = os.path.join(arduino_cli_path, "arduino-cli") + arduino_cli = str(arduino_cli_path / "arduino-cli.exe") else: - arduino_cli = "arduino-cli" + arduino_cli = str(arduino_cli_path / "arduino-cli") try: output = subprocess.check_output( - [arduino_cli, "version"], stderr=subprocess.STDOUT, + [arduino_cli, "version"], + stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as e: print('"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode)) @@ -222,7 +205,14 @@ def check_config(): try: output = subprocess.check_output( - [arduino_cli, "core", "search", "stm32", "--additional-urls", stm32_url], + [ + arduino_cli, + "core", + "search", + "stm32", + "--additional-urls", + stm32_url, + ], stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as e: @@ -249,12 +239,12 @@ def check_config(): cli_config = json.loads(output) if cli_config is not None: if cli_config["directories"]["data"] is not None: - sketches_path_list.append(cli_config["directories"]["data"]) + sketches_path_list.append(Path(cli_config["directories"]["data"])) else: print("No data directory") quit(1) if cli_config["directories"]["user"] is not None: - sketches_path_list.append(cli_config["directories"]["user"]) + sketches_path_list.append(Path(cli_config["directories"]["user"])) else: print("No user directory!") quit(1) @@ -269,9 +259,7 @@ def load_core_config(): global arch cores_config_filename = "" if args.config: - assert os.path.exists( - args.config - ), "User core configuration JSON file does not exist" + assert args.config.exists(), "User core configuration JSON file does not exist" cores_config_filename = args.config else: if args.ci: @@ -379,25 +367,27 @@ def manage_inos(): if args.all or args.sketches or args.list == "sketch": find_inos() if args.exclude: - assert os.path.exists(args.exclude), "Excluded list file does not exist" - manage_exclude_list(args.exclude) - elif os.path.exists(exclude_file_default): + exclude_file = Path(args.exclude) + assert exclude_file.exists(), "Excluded list file does not exist" + manage_exclude_list(exclude_file) + elif exclude_file_default.exists(): manage_exclude_list(exclude_file_default) # Only one sketch elif args.ino: - if os.path.exists(args.ino): + ino_file = Path(args.ino) + if ino_file.exists(): # Store only the path - if os.path.isfile(args.ino): - sketch_list.append(os.path.dirname(args.ino)) + if ino_file.is_file(args.ino): + sketch_list.append(ino_file.parent) else: sketch_list.append(args.ino) else: for path in sketches_path_list: - fp = os.path.join(path, args.ino) - if os.path.exists(fp): + fp = path / ino_file + if fp.exists(): # Store only the path - if os.path.isfile(fp): - sketch_list.append(os.path.dirname(fp)) + if fp.is_file(): + sketch_list.append(fp.parent) else: sketch_list.append(fp) break @@ -406,24 +396,25 @@ def manage_inos(): quit(1) # Sketches listed in a file elif args.file: - assert os.path.exists(args.file), "Sketches list file does not exist" - with open(args.file, "r") as f: + sketches_files = Path(args.file) + assert sketches_files.exists(), "Sketches list file does not exist" + with open(sketches_files, "r") as f: for line in f.readlines(): if line.rstrip(): - ino = line.rstrip() - if os.path.exists(ino): + ino = Path(line.rstrip()) + if ino.exists(): # Store only the path - if os.path.isfile(ino): - sketch_list.append(os.path.dirname(ino)) + if ino.is_file(): + sketch_list.append(ino.parent) else: sketch_list.append(ino) else: for path in sketches_path_list: - fp = os.path.join(path, ino) - if os.path.exists(fp): + fp = path / ino + if fp.exists(): # Store only the path - if os.path.isfile(fp): - sketch_list.append(os.path.dirname(fp)) + if fp.is_file(): + sketch_list.append(fp.parent) else: sketch_list.append(fp) break @@ -443,14 +434,13 @@ def find_inos(): # key: path, value: name if args.sketches: arg_sketch_pattern = re.compile(args.sketches, re.IGNORECASE) - for path in sketches_path_list: - for root, dirs, files in os.walk(path, followlinks=True): - for file in files: - if file.endswith((".ino", ".pde")): - if args.sketches: - if arg_sketch_pattern.search(os.path.join(root, file)) is None: - continue - sketch_list.append(root) + for spath in sketches_path_list: + for spath_object in spath.glob("**/*.[ip][nd][oe]"): + if args.sketches: + if arg_sketch_pattern.search(str(spath_object)) is None: + continue + if spath_object.is_file(): + sketch_list.append(spath_object.parent) sketch_list = sorted(set(sketch_list)) @@ -490,7 +480,16 @@ def find_board(): for fqbn in fqbn_list_tmp: try: output = subprocess.check_output( - [arduino_cli, "board", "details", "--additional-urls", stm32_url, "--format", "json", fqbn], + [ + arduino_cli, + "board", + "details", + "--additional-urls", + stm32_url, + "--format", + "json", + fqbn, + ], stderr=subprocess.STDOUT, ).decode("utf-8") except subprocess.CalledProcessError as e: @@ -528,14 +527,14 @@ def find_board(): def check_status(status, build_conf, boardKo): global nb_build_passed global nb_build_failed - sketch_name = os.path.basename(build_conf[4][-1]) + sketch_name = build_conf[4][-1].name if status[1] == 0: result = "\033[32msucceeded\033[0m" nb_build_passed += 1 elif status[1] == 1: # Check if failed due to a region overflowed - logFile = os.path.join(build_conf[3], sketch_name + ".log") + logFile = build_conf[3] / f"{sketch_name}.log" # error or fatal error fork_pattern = re.compile(r"^Error during build: fork/exec") error_pattern = re.compile(r":\d+:\d+:\s.*error:\s|^Error:") @@ -669,7 +668,7 @@ def log_final_result(): f.write("Duration: ") f.write(duration) f.write("\nLogs are available here:\n") - f.write(output_dir + "\n") + f.write(f"{output_dir}\n") f.write(build_separator + "\n\n") # Standard output @@ -710,17 +709,13 @@ def genBasicCommand(b_name): cmd.append("compile") # cmd.append("-warnings=all") cmd.append("--build-path") - cmd.append(os.path.join(build_output_dir, b_name)) + cmd.append(build_output_dir / b_name) cmd.append("--build-cache-path") cmd.append(build_output_cache_dir) if args.verbose: cmd.append("--verbose") - if version.parse(arduino_cli_version) <= version.parse("0.10.0"): - cmd.append("--output") - cmd.append(os.path.join(output_dir, b_name, bin_dir, "dummy_sketch")) - else: - cmd.append("--output-dir") - cmd.append(os.path.join(output_dir, b_name, bin_dir)) + cmd.append("--output-dir") + cmd.append(output_dir / b_name / bin_dir) cmd.append("--fqbn") cmd.append(get_fqbn(b_name)) cmd.append("dummy_sketch") @@ -736,7 +731,7 @@ def create_build_conf_list(): b_name, idx, len(board_fqbn), - os.path.join(output_dir, b_name), + output_dir / b_name, genBasicCommand(b_name), ) ) @@ -750,13 +745,10 @@ def build_config(sketch, boardSkipped): for idx in reversed(range(len(build_conf_list))): build_conf_list[idx][4][-1] = sketch - build_conf_list[idx][4][-4] = build_conf_list[idx][4][-4].replace( - "dummy_sketch", os.path.basename(sketch) - ) if na_sketch_pattern: if build_conf_list[idx][0] in na_sketch_pattern: for pattern in na_sketch_pattern[build_conf_list[idx][0]]: - if re.search(pattern, sketch, re.IGNORECASE): + if re.search(pattern, str(sketch), re.IGNORECASE): print( (build_format_result).format( "{}/{}".format( @@ -777,7 +769,7 @@ def build_config(sketch, boardSkipped): for pattern in sketch_options: print if pattern in sketch_options: - if re.search(pattern, sketch, re.IGNORECASE): + if re.search(pattern, str(sketch), re.IGNORECASE): if build_conf_list[idx][4][-2].count(":") == 3: build_conf_list[idx][4][-2] += ( "," + sketch_options[pattern] @@ -801,12 +793,10 @@ def build_all(): print(build_separator) print( "| {:^85} |".format( - "Sketch \033[34m{}\033[0m ({}/{})".format( - os.path.basename(sketch), sketch_nb, len(sketch_list) - ) + f"Sketch \033[34m{sketch.name}\033[0m ({sketch_nb}/{len(sketch_list)})" ) ) - wrapped_path_ = wrapper.wrap(text=os.path.dirname(sketch)) + wrapped_path_ = wrapper.wrap(text=str(sketch.parent)) for line in wrapped_path_: print("| {:^76} |".format("{}".format(line))) print(build_separator) @@ -829,9 +819,7 @@ def build_all(): def build(build_conf): cmd = build_conf[4] status = [time.monotonic()] - with open( - os.path.join(build_conf[3], os.path.basename(cmd[-1]) + ".log"), "w" - ) as stdout: + with open(build_conf[3] / f"{cmd[-1].name}.log", "w") as stdout: res = subprocess.Popen(cmd, stdout=stdout, stderr=subprocess.STDOUT) res.wait() status[0] = time.monotonic() - status[0] @@ -877,17 +865,18 @@ def build(build_conf): parser.add_argument( "--config", metavar="", - help="JSON file containing the build configuration for one or more\ + help=f"JSON file containing the build configuration for one or more\ maintainer/architecture. Board options for build, applicability of\ sketches for boards or required options. If sketch is not listed\ - then applicable to all board. Default core configuration is for '" - + arch_default - + " 'architecture in: " - + cores_config_file_default, + then applicable to all board. Default core configuration is for\ + '{arch_default}' architecture in: {cores_config_file_default}", ) parser.add_argument( - "-u", "--url", metavar="", help="additional URL for the board manager\ + "-u", + "--url", + metavar="", + help="additional URL for the board manager\ Default url : " + stm32_url, ) @@ -901,7 +890,7 @@ def build(build_conf): # Sketch options sketchg0 = parser.add_argument_group( - title="Sketch(es) options", description="By default build " + sketch_default + title="Sketch(es) options", description=f"By default build {sketch_default}" ) sketchg1 = sketchg0.add_mutually_exclusive_group() @@ -924,9 +913,8 @@ def build(build_conf): "-e", "--exclude", metavar="", - help="file containing sketches pattern to ignore.\ - Default path : " - + exclude_file_default, + help=f"file containing sketches pattern to ignore.\ + Default path : {exclude_file_default}", ) args = parser.parse_args() diff --git a/CI/utils/pathlib_ext.py b/CI/utils/pathlib_ext.py index de08bd8239..6a7af72ea9 100644 --- a/CI/utils/pathlib_ext.py +++ b/CI/utils/pathlib_ext.py @@ -5,8 +5,7 @@ # Create a folder if not exists def createFolder(path): try: - if not path.exists(): - path.mkdir() + path.mkdir(parents=True, exist_ok=True) except OSError: print("Error: Creating directory {}".format(path)) From 8ff950174a1aa2389ea96973d45a5809d20374e8 Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Fri, 8 Oct 2021 18:52:17 +0200 Subject: [PATCH 03/17] refactor(scripts): use f-string to ease readability Signed-off-by: Frederic Pillon --- CI/build/.flake8 | 3 +- CI/build/arduino-cli.py | 258 ++++++++++++++------------------------ CI/update/.flake8 | 2 +- CI/update/stm32variant.py | 152 ++++++++++------------ CI/update/stm32wrapper.py | 26 ++-- CI/utils/pathlib_ext.py | 7 +- 6 files changed, 183 insertions(+), 265 deletions(-) diff --git a/CI/build/.flake8 b/CI/build/.flake8 index 9b7e20c3a6..c5d8607073 100644 --- a/CI/build/.flake8 +++ b/CI/build/.flake8 @@ -1,2 +1,3 @@ [flake8] -max-line-length = 95 +max-line-length = 90 +ignore = E402 diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index 8afc742f37..bdef414121 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -28,7 +28,8 @@ path_config_filename = script_path / "path_config.json" arduino_cli_path = Path("") -stm32_url = "https://github.com/stm32duino/BoardManagerFiles/raw/main/package_stmicroelectronics_index.json" +stm32_url_base = "https://github.com/stm32duino/BoardManagerFiles/raw/main/" +stm32_url = f"{stm32_url_base}package_stmicroelectronics_index.json" sketches_path_list = [] default_build_output_dir = tempdir / "build_arduinoCliOutput" build_output_dir = tempdir / f"build_arduinoCliOutput{build_id}" @@ -84,10 +85,22 @@ fail_count = 0 skip_count = 0 +# error or fatal error +fork_pattern = re.compile(r"^Error during build: fork/exec") +error_pattern = re.compile(r":\d+:\d+:\s.*error:\s|^Error:") +ld_pattern = re.compile("arm-none-eabi/bin/ld:") +overflow_pattern = re.compile( + r"(will not fit in |section .+ is not within )?region( .+ overflowed by [\d]+ bytes)?" +) + # format build_format_header = "| {:^8} | {:42} | {:^10} | {:^7} |" build_format_result = "| {:^8} | {:42} | {:^19} | {:^6.2f}s |" build_separator = "-" * 80 +fsucc = "\033[32msucceeded\033[0m" +ffail = "\033[31mfailed\033[0m" +fskip = "\033[33mskipped\033[0m" +nl = "\n" def cat(file): @@ -100,10 +113,10 @@ def cat(file): def create_output_log_tree(): # Log output file with open(log_file, "w") as file: - file.write(build_separator + "\nStarts ") + file.write(f"{build_separator}\nStarts ") file.write(time.strftime("%A %d %B %Y %H:%M:%S ")) - file.write(build_separator + "\n") file.write(f"\nLog will be available here:\n{output_dir.resolve()}\n") + file.write(f"{build_separator}\n") # Folders for board in board_fqbn: createFolder(output_dir / board / bin_dir) @@ -117,11 +130,8 @@ def create_config(): global build_output_dir global root_output_dir # Create a Json file for a better path management - print( - "'{}' file created. Please check the configuration.".format( - path_config_filename - ) - ) + print(f"'{path_config_filename}' file created.") + print("Please check the configuration.") path_config_file = open(path_config_filename, "w") path_config_file.write( json.dumps( @@ -186,19 +196,19 @@ def check_config(): stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as e: - print('"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode)) + print(f"'{' '.join(e.cmd)}' failed with code: {e.returncode}!") print(e.stdout.decode("utf-8")) quit(e.returncode) else: res = re.match(r".*Version:\s+(\d+\.\d+\.\d+).*", output.decode("utf-8")) if res: arduino_cli_version = res.group(1) - print("Arduino CLI version used: " + arduino_cli_version) + print(f"Arduino CLI version used: {arduino_cli_version}") + if version.parse(arduino_cli_version) <= version.parse("0.10.0"): + print("Arduino CLI version <= 0.10.0 is no more supported") else: - print( - "Unable to define Arduino CLI version, use default: " - + arduino_cli_default_version - ) + print("Unable to define Arduino CLI version.") + print(f"Use default: {arduino_cli_default_version}") if args.url: stm32_url = args.url @@ -216,12 +226,12 @@ def check_config(): stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as e: - print('"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode)) + print(f"'{' '.join(e.cmd)}' failed with code: {e.returncode}!") print(e.stdout.decode("utf-8")) quit(e.returncode) else: if arduino_platform not in output.decode("utf-8"): - print(arduino_platform + " is not installed!") + print(f"{arduino_platform} is not installed!") quit(1) # Add core and library path to sketches_path_list try: @@ -230,9 +240,7 @@ def check_config(): stderr=subprocess.STDOUT, ).decode("utf-8") except subprocess.CalledProcessError as e: - print( - '"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode) - ) + print(f"'{' '.join(e.cmd)}' failed with code: {e.returncode}!") print(e.stdout.decode("utf-8")) quit(e.returncode) else: @@ -259,7 +267,7 @@ def load_core_config(): global arch cores_config_filename = "" if args.config: - assert args.config.exists(), "User core configuration JSON file does not exist" + assert args.config.exists(), f"{args.config} not found" cores_config_filename = args.config else: if args.ci: @@ -279,35 +287,23 @@ def load_core_config(): if arch == core["architecture"]: core_config = core maintainer = core["maintainer"] - print( - "Build configuration for '" - + maintainer - + "' maintainer and '" - + arch - + "' architecture" - ) + print("Build configuration for:") + print(f"- '{maintainer}' maintainer") + print(f"- '{arch}' architecture") break else: - print( - "Core architecture '" + arch + "' not found in " + cores_config_filename - ) + print(f"Core architecture '{arch}' not found in:") + print(f"{cores_config_filename}") arch = arch_default core_config = None except IOError: - print( - "Can't open {} file. Build configuration will not be used.".format( - cores_config_filename - ) - ) + print(f"Can't open {cores_config_filename} file.") + print("Build configuration will not be used.") finally: if core_config is None: - print( - "Using default configuration for '" - + maintainer_default - + "' maintainer and '" - + arch_default - + "' architecture" - ) + print("Using default configuration for:") + print(f"- '{maintainer_default}' maintainer") + print(f"- '{arch_default}' architecture") # Board list have to be initialized before call this function @@ -347,22 +343,21 @@ def parse_core_config(): # print("{}: {}\n".format(key, value)) -def manage_exclude_list(file): - with open(file, "r") as f: +def manage_exclude_list(exfile): + with open(exfile, "r") as f: for line in f.readlines(): if line.rstrip(): exclude_list.append(line.rstrip()) if exclude_list: for pattern in exclude_list: - exclude_pattern = re.compile(".*" + pattern + ".*", re.IGNORECASE) + exclude_pattern = re.compile(f".*{pattern}.*", re.IGNORECASE) for s in reversed(sketch_list): - if exclude_pattern.search(s): + if exclude_pattern.search(str(s)): sketch_list.remove(s) # Manage sketches list def manage_inos(): - # Find all inos or all patterned inos if args.all or args.sketches or args.list == "sketch": find_inos() @@ -392,12 +387,12 @@ def manage_inos(): sketch_list.append(fp) break else: - print("Sketch {} path does not exist!".format(args.ino)) + print(f"Sketch {args.ino} path does not exist!") quit(1) # Sketches listed in a file elif args.file: sketches_files = Path(args.file) - assert sketches_files.exists(), "Sketches list file does not exist" + assert sketches_files.exists(), f"{sketches_files} not found" with open(sketches_files, "r") as f: for line in f.readlines(): if line.rstrip(): @@ -419,12 +414,12 @@ def manage_inos(): sketch_list.append(fp) break else: - print("Ignore {} as it does not exist.".format(ino)) + print(f"Ignore {ino} as it does not exist.") # Default sketch to build else: sketch_list.append(sketch_default) if len(sketch_list) == 0: - print("No sketch to build for " + arduino_platform + "!") + print(f"No sketch to build for {arduino_platform}!") quit(1) @@ -444,7 +439,8 @@ def find_inos(): sketch_list = sorted(set(sketch_list)) -# Return a list of all board using the arduino-cli for the specified architecture +# Return a list of all board using the arduino-cli for the specified +# architecture def find_board(): global board_fqbn board_found = {} @@ -463,7 +459,7 @@ def find_board(): stderr=subprocess.STDOUT, ).decode("utf-8") except subprocess.CalledProcessError as e: - print('"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode)) + print(f"'{' '.join(e.cmd)}' failed with code: {e.returncode}!") print(e.stdout.decode("utf-8")) quit(e.returncode) else: @@ -473,7 +469,7 @@ def find_board(): if arduino_platform in board[fqbn_key]: fqbn_list_tmp.append(board[fqbn_key]) if not len(fqbn_list_tmp): - print("No boards found for " + arduino_platform) + print(f"No boards found for {arduino_platform}") quit(1) # For STM32 core, pnum is requested @@ -493,9 +489,7 @@ def find_board(): stderr=subprocess.STDOUT, ).decode("utf-8") except subprocess.CalledProcessError as e: - print( - '"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode) - ) + print(f"'{' '.join(e.cmd)}' failed with code: {e.returncode}!") print(e.stdout.decode("utf-8")) quit(e.returncode) else: @@ -510,16 +504,16 @@ def find_board(): if args.board: if arg_board_pattern.search(value["value"]) is None: continue - board_found[value["value"]] = ( - fqbn + ":pnum=" + value["value"] - ) + board_found[ + value["value"] + ] = f"{fqbn}:pnum={value['value']}" break else: - print('No detail found for:"' + fqbn + '"!') + print(f"No detail found for:'{fqbn}'!") if board_found: board_fqbn = collections.OrderedDict(sorted(board_found.items())) else: - print("No board found for " + arduino_platform + "!") + print(f"No board found for {arduino_platform}!") quit(1) @@ -530,18 +524,11 @@ def check_status(status, build_conf, boardKo): sketch_name = build_conf[4][-1].name if status[1] == 0: - result = "\033[32msucceeded\033[0m" + result = fsucc nb_build_passed += 1 elif status[1] == 1: # Check if failed due to a region overflowed logFile = build_conf[3] / f"{sketch_name}.log" - # error or fatal error - fork_pattern = re.compile(r"^Error during build: fork/exec") - error_pattern = re.compile(r":\d+:\d+:\s.*error:\s|^Error:") - ld_pattern = re.compile("arm-none-eabi/bin/ld:") - overflow_pattern = re.compile( - r"(will not fit in |section .+ is not within )?region( .+ overflowed by [\d]+ bytes)?" - ) error_found = False for i, line in enumerate(open(logFile)): if error_pattern.search(line): @@ -553,7 +540,7 @@ def check_status(status, build_conf, boardKo): if overflow_pattern.search(line) is None: error_found = True if error_found: - result = "\033[31mfailed\033[0m" + result = ffail boardKo.append(build_conf[0]) if args.ci: cat(logFile) @@ -567,8 +554,8 @@ def check_status(status, build_conf, boardKo): result = "\033[31merror\033[0m" print( - (build_format_result).format( - "{}/{}".format(build_conf[1], build_conf[2]), + f"{build_format_result}".format( + f"{build_conf[1]}/{build_conf[2]}", build_conf[0], result, status[0], @@ -578,60 +565,35 @@ def check_status(status, build_conf, boardKo): # Log sketch build result def log_sketch_build_result(sketch, boardKo, boardSkipped): + nb_ok = len(board_fqbn) - len(boardKo) - len(boardSkipped) + nb_ko = len(boardKo) + nb_na = len(boardSkipped) # Log file with open(log_file, "a") as f: f.write(build_separator + "\n") - f.write( - "Sketch: {} ({}/{})\n".format( - os.path.basename(sketch), - sketch_list.index(sketch) + 1, - len(sketch_list), - ) - ) - f.write(os.path.dirname(sketch)) - f.write( - "\n{} {}, {} {}, {} {}\n".format( - len(board_fqbn) - len(boardKo) - len(boardSkipped), - "succeeded", - len(boardKo), - "failed", - len(boardSkipped), - "skipped", - ) - ) + bcounter = f"{sketch_list.index(sketch) + 1}/{len(sketch_list)}" + f.write(f"Sketch: {sketch.name} ({bcounter})\n") + f.write(str(sketch.parent)) + f.write(f"\n{nb_ok} succeeded, {nb_ko} failed, {nb_na} skipped\n") if len(boardKo): - f.write( - "Failed boards :\n" - + "\n".join( - textwrap.wrap(", ".join(boardKo), 80, break_long_words=False) - ) - ) + sKo = ", ".join(boardKo) + wKo = textwrap.wrap(sKo, 80, break_long_words=False) + f.write(f"Failed boards :\n{nl.join(wKo)}") # f.write("Failed boards :\n" + "\n".join(boardKo)) f.write("\n") if len(boardSkipped): - f.write( - "Skipped boards :\n" - + "\n".join( - textwrap.wrap(", ".join(boardSkipped), 80, break_long_words=False) - ) - ) + sskipped = ", ".join(boardSkipped) + wskipped = textwrap.wrap(sskipped, 80, break_long_words=False) + f.write(f"Skipped boards :\n{nl.join(wskipped)}") # f.write("Skipped boards :\n" + "\n".join(boardSkipped)) f.write("\n") - f.write(build_separator + "\n") + f.write(f"{build_separator}\n") # Standard output print(build_separator) - print( - "Build Summary: {} {}, {} {}, {} {}".format( - len(board_fqbn) - len(boardKo) - len(boardSkipped), - "\033[32msucceeded\033[0m", - len(boardKo), - "\033[31mfailed\033[0m", - len(boardSkipped), - "\033[33mskipped\033[0m", - ) - ) + + print(f"Build Summary: {nb_ok} {fsucc}, {nb_ko} {ffail}, {nb_na} {fskip}") print(build_separator) @@ -646,47 +608,24 @@ def log_final_result(): # Log file with open(log_file, "a") as f: - f.write("\n" + build_separator + "\n") + f.write(f"\n{build_separator}\n") f.write("| {:^76} |\n".format("Build summary")) f.write(build_separator + "\n") - f.write( - "{} {} ({}%), {} {} ({}%), {} {} ({}%) of {} builds\n".format( - nb_build_passed, - "succeeded", - stat_passed, - nb_build_failed, - "failed", - stat_failed, - nb_build_skipped, - "skipped", - stat_skipped, - nb_build_total, - ) - ) - f.write("Ends ") - f.write(time.strftime("%A %d %B %Y %H:%M:%S\n")) - f.write("Duration: ") - f.write(duration) - f.write("\nLogs are available here:\n") - f.write(f"{output_dir}\n") - f.write(build_separator + "\n\n") + ssucc = f"{nb_build_passed} succeeded ({stat_passed}%)" + sfail = f"{nb_build_failed} failed ({stat_failed}%)" + sskip = f"{nb_build_skipped} skipped ({stat_skipped}%)" + f.write(f"{ssucc}, {sfail}, {sskip} of {nb_build_total} builds\n") + f.write(f"Ends {time.strftime('%A %d %B %Y %H:%M:%S')}\n") + f.write(f"Duration: {duration}\n") + f.write(f"Logs are available here:\n{output_dir}\n") + f.write(f"{build_separator}\n\n") # Standard output - print( - "Builds Summary: {} {} ({}%), {} {} ({}%), {} {} ({}%) of {} builds".format( - nb_build_passed, - "\033[32msucceeded\033[0m", - stat_passed, - nb_build_failed, - "\033[31mfailed\033[0m", - stat_failed, - nb_build_skipped, - "\033[33mskipped\033[0m", - stat_skipped, - nb_build_total, - ) - ) - print("Duration: " + duration) + ssucc = f"{nb_build_passed} {fsucc} ({stat_passed}%)" + sfail = f"{nb_build_failed} {ffail} ({stat_failed}%)" + sskip = f"{nb_build_skipped} {fskip} ({stat_skipped}%)" + print(f"Builds Summary: {ssucc}, {sfail}, {sskip} of {nb_build_total} builds") + print(f"Duration: {duration}") print("Logs are available here:") print(output_dir) @@ -755,7 +694,7 @@ def build_config(sketch, boardSkipped): build_conf_list[idx][1], build_conf_list[idx][2] ), build_conf_list[idx][0], - "\033[33mskipped\033[0m", + fskip, 0.00, ) ) @@ -798,7 +737,7 @@ def build_all(): ) wrapped_path_ = wrapper.wrap(text=str(sketch.parent)) for line in wrapped_path_: - print("| {:^76} |".format("{}".format(line))) + print("| {:^76} |".format(f"{line}")) print(build_separator) print((build_format_header).format("Num", "Board", "Result", "Time")) print(build_separator) @@ -858,9 +797,7 @@ def build(build_conf): parser.add_argument( "--arch", metavar="architecture", - help="core architecture to build. Default build architecture is '" - + arch_default - + "'", + help=f"core architecture to build. Default build architecture is {arch_default}", ) parser.add_argument( "--config", @@ -876,9 +813,8 @@ def build(build_conf): "-u", "--url", metavar="", - help="additional URL for the board manager\ - Default url : " - + stm32_url, + help=f"additional URL for the board manager\ + Default url : {stm32_url}", ) parser.add_argument( @@ -929,16 +865,16 @@ def main(): load_core_config() find_board() if args.list == "board": - print("%i board(s) available" % len(board_fqbn)) for board in board_fqbn: print(board) + print(f"{len(board_fqbn)} board(s) available") quit() manage_inos() if args.list == "sketch": for sketch in sketch_list: print(sketch) - print("%i sketches found" % len(sketch_list)) + print(f"{len(sketch_list)} sketches found") quit() if core_config: diff --git a/CI/update/.flake8 b/CI/update/.flake8 index d74cc78fd4..c55e852d7d 100644 --- a/CI/update/.flake8 +++ b/CI/update/.flake8 @@ -1,3 +1,3 @@ [flake8] max-line-length = 95 -ignore = W503, E203 +ignore = W503, E203, E402 diff --git a/CI/update/stm32variant.py b/CI/update/stm32variant.py index a30476011a..1bcfb6b702 100644 --- a/CI/update/stm32variant.py +++ b/CI/update/stm32variant.py @@ -863,25 +863,25 @@ def qspi_pinmap(lst): name = "OCTOSPI" hal = "OSPI" if lst == quadspidata0_list: - aname = name + "_DATA0" + aname = f"{name}_DATA0" elif lst == quadspidata1_list: - aname = name + "_DATA1" + aname = f"{name}_DATA1" elif lst == quadspidata2_list: - aname = name + "_DATA2" + aname = f"{name}_DATA2" elif lst == quadspidata3_list: - aname = name + "_DATA3" + aname = f"{name}_DATA3" elif lst == quadspidata4_list: - aname = name + "_DATA4" + aname = f"{name}_DATA4" elif lst == quadspidata5_list: - aname = name + "_DATA5" + aname = f"{name}_DATA5" elif lst == quadspidata6_list: - aname = name + "_DATA6" + aname = f"{name}_DATA6" elif lst == quadspidata7_list: - aname = name + "_DATA7" + aname = f"{name}_DATA7" elif lst == quadspisclk_list: - aname = name + "_SCLK" + aname = f"{name}_SCLK" else: - aname = name + "_SSEL" + aname = f"{name}_SSEL" for p in lst: # 2nd element is the XXXXSPI_YYYY signal instm = re.match(ospi_regex, p[2]) @@ -1093,7 +1093,7 @@ def manage_syswkup(): num -= 1 cmt = "" else: - cmt = " /* " + p[2] + " */" + cmt = f" /* {p[2]} */" syswkup_pins_list[num].append([p[0], cmt]) return syswkup_pins_list @@ -1144,7 +1144,7 @@ def print_pinamevar(): for idx, syswkup_list in enumerate(syswkup_pins_list, start=1): if len(syswkup_list) > 1: for idx2, lst in enumerate(syswkup_list[1:], start=1): - alt_syswkup_list.append("{}_{}".format(idx, idx2)) + alt_syswkup_list.append(f"{idx}_{idx2}") return alt_syswkup_list @@ -1246,7 +1246,7 @@ def serial_pins_variant(): if serialnum: serialnum = serialnum.group(1) if serial_inst.startswith("LP"): - serialnum = "10" + serialnum + serialnum = f"10{serialnum}" else: print("No serial instance number found!") serialnum = "-1" @@ -1302,8 +1302,8 @@ def print_variant(generic_list, alt_syswkup_list): for idx, io in enumerate(io_list): pyn = io[0].replace("_", "", 1) if [item for item in adclist if item[0] == io[0]]: - ax = "A{}".format(analog_index) - pins_number_list.append({"name": pyn, "val": "PIN_" + ax}) + ax = f"A{analog_index}" + pins_number_list.append({"name": pyn, "val": f"PIN_{ax}"}) pinnames_list.append({"name": io[0], "ax": analog_index}) analog_pins_list.append({"val": idx, "ax": ax, "pyn": pyn}) analog_index += 1 @@ -1314,8 +1314,8 @@ def print_variant(generic_list, alt_syswkup_list): for idx, io in enumerate(dualpad_list): pyn = io[0].replace("_", "", 1) if [item for item in adclist if item[0] == io[0]]: - ax = "A{}".format(analog_index) - pins_number_list.append({"name": pyn, "val": "PIN_" + ax}) + ax = f"A{analog_index}" + pins_number_list.append({"name": pyn, "val": f"PIN_{ax}"}) pinnames_list.append({"name": io[0], "ax": analog_index}) analog_pins_list.append({"val": idx + idx_sum, "ax": ax, "pyn": pyn}) analog_index += 1 @@ -1326,8 +1326,8 @@ def print_variant(generic_list, alt_syswkup_list): for idx, io in enumerate(remap_list): pyn = io[0].replace("_", "", 1) if [item for item in adclist if item[0] == io[0]]: - ax = "A{}".format(analog_index) - pins_number_list.append({"name": pyn, "val": "PIN_" + ax}) + ax = f"A{analog_index}" + pins_number_list.append({"name": pyn, "val": f"PIN_{ax}"}) pinnames_list.append({"name": io[0], "ax": analog_index}) analog_pins_list.append({"val": idx + idx_sum, "ax": ax, "pyn": pyn}) analog_index += 1 @@ -1822,7 +1822,7 @@ def group_by_flash(group_base_list, glist, index_mcu_base): ] # Merge key if key: - packages_per_flash[key + "-" + flash] = packages_per_flash.pop(key) + packages_per_flash[f"{key}-{flash}"] = packages_per_flash.pop(key) else: packages_per_flash[flash] = packages_list @@ -1832,7 +1832,7 @@ def group_by_flash(group_base_list, glist, index_mcu_base): if len(key) == 1: new_mcu_dirname += key else: - new_mcu_dirname += "(" + key + ")" + new_mcu_dirname += f"({key})" # Handle package with ANPQX # One case not manage: [Tx, TxX, Yx] # Assuming it is not an issue to have non existing mcu @@ -1845,9 +1845,7 @@ def group_by_flash(group_base_list, glist, index_mcu_base): # Assert if sub.group(2) != "x": print( - "Package of {} info contains {} instead of 'x'".format( - base_name, sub.group(2) - ) + "Package of {base_name} info contains {sub.group(2)} instead of 'x'" ) exit(1) if sub.group(3): @@ -1858,7 +1856,7 @@ def group_by_flash(group_base_list, glist, index_mcu_base): if len(pcounter) == 1: new_mcu_dirname += package_list[0] else: - new_mcu_dirname += "(" + "-".join(k for k in sorted(pcounter)) + ")" + new_mcu_dirname += f"({'-'.join(k for k in sorted(pcounter))})" if len(ecounter): new_mcu_dirname += "x" if (len(ecounter) == 1) and ( @@ -1867,7 +1865,7 @@ def group_by_flash(group_base_list, glist, index_mcu_base): # new_mcu_dirname += next(iter(ecounter)) new_mcu_dirname += ext_list[0] else: - new_mcu_dirname += "(" + "-".join(k for k in sorted(ecounter)) + ")" + new_mcu_dirname += f"({'-'.join(k for k in sorted(ecounter))})" del package_list[:] del ext_list[:] @@ -1903,12 +1901,11 @@ def merge_dir(out_temp_path, group_mcu_dir, mcu_family, periph_xml, variant_exp) for index, glist in enumerate(group_base_list): # Only one mcu if len(glist) == 1: - new_mcu_dirname += ("_" if index != 0 else "") + glist[0].strip("x") + new_mcu_dirname += f"{'_' if index != 0 else ''}{glist[0].strip('x')}" else: # Group using flash info - new_mcu_dirname += ("_" if index != 0 else "") + group_by_flash( - group_base_list, glist, index_mcu_base - ) + gbf = group_by_flash(group_base_list, glist, index_mcu_base) + new_mcu_dirname += f"{'_' if index != 0 else ''}{gbf}" del group_package_list[:] del group_flash_list[:] del group_base_list[:] @@ -1951,23 +1948,23 @@ def merge_dir(out_temp_path, group_mcu_dir, mcu_family, periph_xml, variant_exp) new_line_c = periph_xml.pop(0) for index, xml in enumerate(periph_xml, 1): if index % 2 == 0: - new_line_c += "\n * {}".format(xml) + new_line_c += f"\n * {xml}" else: - new_line_c += ", {}".format(xml) + new_line_c += f", {xml}" update_file(mcu_dir / periph_c_filename, periperalpins_regex, new_line_c) variant_exp.sort() variant_exp = list(OrderedDict.fromkeys(variant_exp)) new_line_c = variant_exp[0] - new_line_h = "{}".format(variant_exp.pop(0)) + new_line_h = f"{variant_exp.pop(0)}" for index, pre in enumerate(variant_exp, 1): if index % 2 == 0: - new_line_c += " ||\\\n {}".format(pre) - new_line_h += " &&\\\n !{}".format(pre) + new_line_c += f" ||\\\n {pre}" + new_line_h += f" &&\\\n !{pre}" else: - new_line_c += " || {}".format(pre) - new_line_h += " && !{}".format(pre) + new_line_c += f" || {pre}" + new_line_h += f" && !{pre}" update_file(mcu_dir / variant_cpp_filename, update_regex, new_line_c) update_file(mcu_dir / generic_clock_filename, update_regex, new_line_c) update_file(mcu_dir / variant_h_filename, update_regex, new_line_h) @@ -2102,7 +2099,7 @@ def default_cubemxdir(): def create_config(): # Create a Json file for a better path management try: - print("Please set your configuration in '{}' file".format(config_filename)) + print(f"Please set your configuration in '{config_filename}' file") config_file = open(config_filename, "w", newline="\n") config_file.write( json.dumps( @@ -2115,7 +2112,7 @@ def create_config(): ) config_file.close() except IOError: - print("Failed to open " + config_filename) + print(f"Failed to open {config_filename}") exit(1) @@ -2141,7 +2138,7 @@ def check_config(): if conf: cubemxdir = Path(conf) except IOError: - print("Failed to open " + config_filename) + print(f"Failed to open {config_filename}") else: create_config() @@ -2152,7 +2149,7 @@ def manage_repo(): try: if not args.skip: - print("Updating " + repo_name + "...") + print(f"Updating {repo_name}...") if repo_path.is_dir(): # Get new tags from the remote git_cmds = [ @@ -2189,7 +2186,7 @@ def manage_repo(): db_release = version_tag return True except subprocess.CalledProcessError as e: - print("Command {} failed with error code {}".format(e.cmd, e.returncode)) + print(f"Command {e.cmd} failed with error code {e.returncode}") return False @@ -2221,32 +2218,22 @@ def manage_repo(): # By default, generate for all mcu xml files description parser = argparse.ArgumentParser( description=textwrap.dedent( - """\ + f""" By default, generates: - - {}, - - {}, - - {}, - - {}, - - {} - - {} + - {periph_c_filename}, + - {pinvar_h_filename}, + - {variant_cpp_filename}, + - {variant_h_filename}, + - {boards_entry_filename} + - {generic_clock_filename} for all xml files description available in STM32CubeMX internal database. -Internal database path must be defined in {}. +Internal database path must be defined in {config_filename}. It can be the one from STM32CubeMX directory if defined: -\t{} +\t{cubemxdir} or the one from GitHub: -\t{} - -""".format( - periph_c_filename, - pinvar_h_filename, - variant_cpp_filename, - variant_h_filename, - boards_entry_filename, - generic_clock_filename, - config_filename, - cubemxdir, - gh_url, - ) +\t{gh_url} + +""" ), epilog=textwrap.dedent( """\ @@ -2287,18 +2274,16 @@ def manage_repo(): "-c", "--cube", help=textwrap.dedent( - """\ -Use STM32CubeMX internal database. Default use GitHub {} repository. -""".format( - repo_name - ) + f"""\ +Use STM32CubeMX internal database. Default use GitHub {repo_name} repository. +""" ), action="store_true", ) parser.add_argument( "-s", "--skip", - help="Skip {} clone/fetch".format(repo_name), + help=f"Skip {repo_name} clone/fetch", action="store_true", ) args = parser.parse_args() @@ -2315,12 +2300,10 @@ def manage_repo(): if fallback or args.cube: if not (cubemxdir.is_dir()): print( - """ + f""" Cube Mx seems not to be installed or not at the specified location. -Please check the value set for 'CUBEMX_DIRECTORY' in '{}' file.""".format( - config_filename - ) +Please check the value set for 'CUBEMX_DIRECTORY' in '{config_filename}' file.""" ) quit() @@ -2339,7 +2322,7 @@ def manage_repo(): release_match = re.match(release_regex, db_release) if release_match: db_release = release_match.group(1) -print("CubeMX DB release {}\n".format(db_release)) +print(f"CubeMX DB release {db_release}\n") # if args.mcu: # # Check input file exists @@ -2382,11 +2365,11 @@ def manage_repo(): xml_mcu.unlink() continue - print("Generating files for '{}'...".format(mcu_file.name)) + print(f"Generating files for '{mcu_file.name}'...") if not gpiofile: print("Could not find GPIO file") quit() - xml_gpio = parse(str(dirIP / ("GPIO-" + gpiofile + "_Modes.xml"))) + xml_gpio = parse(str(dirIP / f"GPIO-{gpiofile}_Modes.xml")) mcu_family_dir = mcu_family + "xx" out_temp_path = tmp_dir / mcu_family_dir / mcu_file.stem.replace("STM32", "") @@ -2415,17 +2398,14 @@ def manage_repo(): print_variant(generic_list, alt_syswkup_list) del alt_syswkup_list[:] del generic_list[:] - print( - "* Total I/O pins found: {}".format( - len(io_list) + len(alt_list) + len(dualpad_list) + len(remap_list) - ) - ) - print(" - {} I/O pins".format(len(io_list))) + sum_io = len(io_list) + len(alt_list) + len(dualpad_list) + len(remap_list) + print(f"* Total I/O pins found: {sum_io}") + print(f" - {len(io_list)} I/O pins") if len(dualpad_list): - print(" - {} dual pad".format(len(dualpad_list))) + print(f" - {len(dualpad_list)} dual pad") if len(remap_list): - print(" - {} remap pins".format(len(remap_list))) - print(" - {} ALT I/O pins".format(len(alt_list))) + print(f" - {len(remap_list)} remap pins") + print(f" - {len(alt_list)} ALT I/O pins") # for io in io_list: # print(io[0] + ", " + io[1]) diff --git a/CI/update/stm32wrapper.py b/CI/update/stm32wrapper.py index d0278e14de..04d90a9e06 100644 --- a/CI/update/stm32wrapper.py +++ b/CI/update/stm32wrapper.py @@ -83,7 +83,7 @@ def checkConfig(arg_core, arg_cmsis): CMSIS_path = core_path.parent / "ArduinoModule-CMSIS" / "CMSIS_5" if not core_path.is_dir(): - print("Could not find " + core_path) + print(f"Could not find {core_path}") exit(1) system_path = core_path / "system" @@ -109,7 +109,7 @@ def printCMSISStartup(log): filelist = [pth.name for pth in filelist] if len(filelist): if log: - print("Number of startup files: {}".format(len(filelist))) + print(f"Number of startup files: {len(filelist)}") # Some mcu have two startup files # Ex: WL one for cm0plus and one for cm4 # In that case this is the same value line so add an extra defined @@ -141,7 +141,7 @@ def printSystemSTM32(log): filelist = sorted(system_path.glob("STM32*/system_stm32*.c")) if len(filelist): if log: - print("Number of system stm32 files: {}".format(len(filelist))) + print(f"Number of system stm32 files: {len(filelist)}") system_list = [] for fp in filelist: system_list.append({"serie": fp.parent.name, "fn": fp.name}) @@ -175,16 +175,16 @@ def wrap(arg_core, arg_cmsis, log): hal_c_dict = {} # Search all files for each series for serie in stm32_series: - src = HALDrivers_path / ("STM32" + serie + "xx_HAL_Driver") / "Src" - inc = HALDrivers_path / ("STM32" + serie + "xx_HAL_Driver") / "Inc" + src = HALDrivers_path / f"STM32{serie}xx_HAL_Driver" / "Src" + inc = HALDrivers_path / f"STM32{serie}xx_HAL_Driver" / "Inc" if src.exists(): if log: - print("Generating for " + serie + "...") + print(f"Generating for {serie}...") lower = serie.lower() # Search stm32yyxx_[hal|ll]*.c file - filelist = src.glob("stm32" + lower + "xx_*.c") + filelist = src.glob(f"stm32{lower}xx_*.c") for fp in filelist: # File name fn = fp.name @@ -204,7 +204,7 @@ def wrap(arg_core, arg_cmsis, log): hal_c_dict[peripheral] = [lower] # Search stm32yyxx_ll_*.h file - filelist = inc.glob("stm32" + lower + "xx_ll_*.h") + filelist = inc.glob(f"stm32{lower}xx_ll_*.h") for fp in filelist: # File name fn = fp.name @@ -267,7 +267,7 @@ def wrap(arg_core, arg_cmsis, log): # CMSIS DSP C source file if not CMSIS_path.is_dir(): - print("Could not find {}".format(CMSIS_path)) + print(f"Could not find {CMSIS_path}") print("CMSIS DSP generation skipped.") else: # Delete all subfolders @@ -282,7 +282,7 @@ def wrap(arg_core, arg_cmsis, log): fdn = CMSIS_DSP_outSrc_path / dn if not fdn.is_dir(): createFolder(fdn) - out_file = open(fdn / (dn + ".c"), "w", newline="\n") + out_file = open(fdn / (f"{dn}.c"), "w", newline="\n") all_ll_file.write(dsp_file_template.render(dsp_path=dn)) out_file.close() return 0 @@ -291,19 +291,19 @@ def wrap(arg_core, arg_cmsis, log): if __name__ == "__main__": # Parser wrapparser = argparse.ArgumentParser( - description="Generate all wrappers files need by the STM32 core (HAL, LL, CMSIS, ...)" + description="Generate all wrappers files (HAL, LL, CMSIS, ...)" ) wrapparser.add_argument( "-c", "--core", metavar="core_path", - help="Root path of the STM32 core. Default: {}".format(core_path), + help=f"Root path of the STM32 core. Default: {core_path}", ) wrapparser.add_argument( "-s", "--cmsis", metavar="cmsis_path", - help="Root path of the CMSIS. Default: {}".format(CMSIS_path), + help=f"Root path of the CMSIS. Default: {CMSIS_path}", ) wrapargs = wrapparser.parse_args() diff --git a/CI/utils/pathlib_ext.py b/CI/utils/pathlib_ext.py index 6a7af72ea9..27134f2624 100644 --- a/CI/utils/pathlib_ext.py +++ b/CI/utils/pathlib_ext.py @@ -1,5 +1,6 @@ import re import shutil +import sys # Create a folder if not exists @@ -7,7 +8,7 @@ def createFolder(path): try: path.mkdir(parents=True, exist_ok=True) except OSError: - print("Error: Creating directory {}".format(path)) + print(f"Error: Creating directory {path}") # Delete targeted folder recursively @@ -22,7 +23,7 @@ def copyFolder(src, dest, ign_patt=set()): if src.is_dir(): shutil.copytree(src, dest, ignore=shutil.ignore_patterns(*ign_patt)) except OSError as e: - print("Error: Folder {} not copied. {}".format(src, e)) + print(f"Error: Folder {src} not copied. {e}") # copy one file to dest @@ -31,7 +32,7 @@ def copyFile(src, dest): if src.is_file(): shutil.copy(str(src), str(dest)) except OSError as e: - print("Error: File {} not copied. {}".format(src, e)) + print(f"Error: File {src} not copied. {e}") def genSTM32List(path, pattern): From 0e05e8c62e7136cdf9fc397322766e98df814a7a Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Thu, 14 Oct 2021 17:11:35 +0200 Subject: [PATCH 04/17] fix(arduino_cli): missing failed board count Signed-off-by: Frederic Pillon --- CI/build/arduino-cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index bdef414121..6604b804e4 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -552,7 +552,8 @@ def check_status(status, build_conf, boardKo): nb_build_passed += 1 else: result = "\033[31merror\033[0m" - + boardKo.append(build_conf[0]) + nb_build_failed += 1 print( f"{build_format_result}".format( f"{build_conf[1]}/{build_conf[2]}", From 6701790ae218fa7cb5aea794e91879086e01af99 Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Thu, 14 Oct 2021 17:12:28 +0200 Subject: [PATCH 05/17] ci(arduino_cli): add a dry run option (no build) Signed-off-by: Frederic Pillon --- CI/build/arduino-cli.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index 6604b804e4..47063aa821 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -6,6 +6,7 @@ import os from packaging import version from pathlib import Path +import random import re import subprocess import sys @@ -725,6 +726,11 @@ def build_config(sketch, boardSkipped): def build_all(): create_output_log_tree() wrapper = textwrap.TextWrapper(width=76) + if args.dry: + print("Performing a dry run (no build)") + build = dry_build + else: + build = real_build for sketch_nb, sketch in enumerate(sketch_list, start=1): boardKo = [] @@ -756,7 +762,7 @@ def build_all(): # Run arduino-cli command -def build(build_conf): +def real_build(build_conf): cmd = build_conf[4] status = [time.monotonic()] with open(build_conf[3] / f"{cmd[-1].name}.log", "w") as stdout: @@ -767,6 +773,23 @@ def build(build_conf): return status +# Run arduino-cli command +def dry_build(build_conf): + # cmd = build_conf[4] + status = [random.random() * 10, random.randint(0, 1)] + if status[1] == 1: + # Create dummy log file + logFile = build_conf[3] / f"{ build_conf[4][-1].name}.log" + # random failed + dummy = open(logFile, "w") + if random.randint(0, 1) == 1: + dummy.writelines("region `FLASH' overflowed by 5612 bytes") + else: + dummy.writelines("Error:") + dummy.close() + return status + + # Parser parser = argparse.ArgumentParser( description="Manage arduino-cli to build sketche(s) for STM32 boards." @@ -795,6 +818,11 @@ def build(build_conf): parser.add_argument( "-c", "--clean", help="clean output directory.", action="store_true" ) + +parser.add_argument( + "-d", "--dry", help="perform a dry run (no build)", action="store_true" +) + parser.add_argument( "--arch", metavar="architecture", From 0a6e5e4012b44ba6b58d868f4a6b1dee7e3f94b1 Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Thu, 14 Oct 2021 17:38:17 +0200 Subject: [PATCH 06/17] ci(arduino_cli): skip find board when sketch is required Signed-off-by: Frederic Pillon --- CI/build/arduino-cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index 47063aa821..4d34ba9995 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -892,7 +892,8 @@ def main(): deleteFolder(root_output_dir) load_core_config() - find_board() + if args.list != "sketch": + find_board() if args.list == "board": for board in board_fqbn: print(board) From 31018a93c9f347f2c8b4cc22f65be6059a55654b Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Fri, 15 Oct 2021 14:55:50 +0200 Subject: [PATCH 07/17] ci(arduino_cli): avoid search on same base path Signed-off-by: Frederic Pillon --- CI/build/arduino-cli.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index 4d34ba9995..94864b364f 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -32,6 +32,7 @@ stm32_url_base = "https://github.com/stm32duino/BoardManagerFiles/raw/main/" stm32_url = f"{stm32_url_base}package_stmicroelectronics_index.json" sketches_path_list = [] +search_path_list = [] default_build_output_dir = tempdir / "build_arduinoCliOutput" build_output_dir = tempdir / f"build_arduinoCliOutput{build_id}" build_output_cache_dir = build_output_dir / "cache" @@ -154,6 +155,7 @@ def check_config(): global arduino_cli_version global arduino_cli_path global sketches_path_list + global search_path_list global build_output_dir global root_output_dir global output_dir @@ -257,6 +259,13 @@ def check_config(): else: print("No user directory!") quit(1) + # Fill search_path_list to avoid search on the same path + sorted_spl = sorted(set(sketches_path_list)) + search_path_list = [] + while sorted_spl: + p = sorted_spl.pop(0) + if not any(root in p.parents for root in search_path_list): + search_path_list.append(Path(p)) else: print("No arduino-cli config!") quit(1) @@ -430,7 +439,7 @@ def find_inos(): # key: path, value: name if args.sketches: arg_sketch_pattern = re.compile(args.sketches, re.IGNORECASE) - for spath in sketches_path_list: + for spath in search_path_list: for spath_object in spath.glob("**/*.[ip][nd][oe]"): if args.sketches: if arg_sketch_pattern.search(str(spath_object)) is None: From 5d2bae4b3582f0d1ce033c17271cf103a3327940 Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Fri, 15 Oct 2021 15:49:25 +0200 Subject: [PATCH 08/17] ci(arduino_cli): support version higher or equal to 0.19.0 Signed-off-by: Frederic Pillon --- CI/build/arduino-cli.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index 94864b364f..1623d78c6f 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -59,8 +59,8 @@ arch = arch_default arduino_platform = arduino_platform_default arduino_cli = "" -arduino_cli_default_version = "0.18.0" -arduino_cli_version = arduino_cli_default_version +arduino_cli_default_ver = "0.19.0" +arduino_cli_ver = arduino_cli_default_ver # List sketch_list = [] @@ -152,7 +152,7 @@ def create_config(): def check_config(): global arduino_cli - global arduino_cli_version + global arduino_cli_ver global arduino_cli_path global sketches_path_list global search_path_list @@ -205,13 +205,13 @@ def check_config(): else: res = re.match(r".*Version:\s+(\d+\.\d+\.\d+).*", output.decode("utf-8")) if res: - arduino_cli_version = res.group(1) - print(f"Arduino CLI version used: {arduino_cli_version}") - if version.parse(arduino_cli_version) <= version.parse("0.10.0"): - print("Arduino CLI version <= 0.10.0 is no more supported") + arduino_cli_ver = res.group(1) + print(f"Arduino CLI version used: {arduino_cli_ver}") + if version.parse(arduino_cli_ver) < version.parse(arduino_cli_default_ver): + print(f"Arduino CLI version < {arduino_cli_default_ver} not supported") else: print("Unable to define Arduino CLI version.") - print(f"Use default: {arduino_cli_default_version}") + print(f"Use default: {arduino_cli_default_ver}") if args.url: stm32_url = args.url @@ -458,10 +458,6 @@ def find_board(): if args.board: arg_board_pattern = re.compile(args.board, re.IGNORECASE) - if version.parse(arduino_cli_version) >= version.parse("0.18.0"): - fqbn_key = "fqbn" - else: - fqbn_key = "FQBN" fqbn_list_tmp = [] try: output = subprocess.check_output( @@ -476,8 +472,8 @@ def find_board(): boards_list = json.loads(output) if boards_list is not None: for board in boards_list["boards"]: - if arduino_platform in board[fqbn_key]: - fqbn_list_tmp.append(board[fqbn_key]) + if arduino_platform in board["fqbn"]: + fqbn_list_tmp.append(board["fqbn"]) if not len(fqbn_list_tmp): print(f"No boards found for {arduino_platform}") quit(1) From c9aaab17236801b02f7268d794c85aecc13e5224 Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Fri, 15 Oct 2021 19:11:37 +0200 Subject: [PATCH 09/17] ci(arduino_cli): review find board function Signed-off-by: Frederic Pillon --- CI/build/arduino-cli.py | 38 +++++++++++++++----------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index 1623d78c6f..b29817f760 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -458,10 +458,9 @@ def find_board(): if args.board: arg_board_pattern = re.compile(args.board, re.IGNORECASE) - fqbn_list_tmp = [] try: output = subprocess.check_output( - [arduino_cli, "board", "listall", "--format", "json"], + [arduino_cli, "board", "search", arduino_platform, "--format", "json"], stderr=subprocess.STDOUT, ).decode("utf-8") except subprocess.CalledProcessError as e: @@ -469,11 +468,7 @@ def find_board(): print(e.stdout.decode("utf-8")) quit(e.returncode) else: - boards_list = json.loads(output) - if boards_list is not None: - for board in boards_list["boards"]: - if arduino_platform in board["fqbn"]: - fqbn_list_tmp.append(board["fqbn"]) + fqbn_list_tmp = [board["fqbn"] for board in json.loads(output)] if not len(fqbn_list_tmp): print(f"No boards found for {arduino_platform}") quit(1) @@ -490,6 +485,7 @@ def find_board(): stm32_url, "--format", "json", + "-b", fqbn, ], stderr=subprocess.STDOUT, @@ -500,22 +496,18 @@ def find_board(): quit(e.returncode) else: board_detail = json.loads(output) - if board_detail is not None: - if "config_options" not in board_detail: - print("No config_options found for " + fqbn) - quit(1) - for option in board_detail["config_options"]: - if option["option"] == "pnum": - for value in option["values"]: - if args.board: - if arg_board_pattern.search(value["value"]) is None: - continue - board_found[ - value["value"] - ] = f"{fqbn}:pnum={value['value']}" - break - else: - print(f"No detail found for:'{fqbn}'!") + for val in next( + ( + item + for item in board_detail.get("config_options", []) + if item["option"] == "pnum" + ), + {}, + ).get("values", []): + if args.board: + if arg_board_pattern.search(val["value"]) is None: + continue + board_found[val["value"]] = f"{fqbn}:pnum={val['value']}" if board_found: board_fqbn = collections.OrderedDict(sorted(board_found.items())) else: From 8acf32e8f25f1a15eaa2ab4350191932350dfb57 Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Mon, 18 Oct 2021 17:49:56 +0200 Subject: [PATCH 10/17] ci(arduino_cli): workaround to follow simlink Due to issue with Path.glob() which does not follow symlink use glob.glob See: https://bugs.python.org/issue33428 Signed-off-by: Frederic Pillon --- CI/build/arduino-cli.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index b29817f760..bb9e97410e 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -2,6 +2,7 @@ import collections import concurrent.futures from datetime import timedelta +from glob import glob import json import os from packaging import version @@ -440,12 +441,22 @@ def find_inos(): if args.sketches: arg_sketch_pattern = re.compile(args.sketches, re.IGNORECASE) for spath in search_path_list: - for spath_object in spath.glob("**/*.[ip][nd][oe]"): + # Due to issue with Path.glob() which does not follow symlink + # use glob.glob + # See: https://bugs.python.org/issue33428 + # for spath_object in spath.glob("**/*.[ip][nd][oe]"): + # if args.sketches: + # if arg_sketch_pattern.search(str(spath_object)) is None: + # continue + # if spath_object.is_file(): + # sketch_list.append(spath_object.parent) + for sk_ino in glob(str(spath / "**" / "*.[ip][nd][oe]"), recursive=True): if args.sketches: - if arg_sketch_pattern.search(str(spath_object)) is None: + if arg_sketch_pattern.search(sk_ino) is None: continue - if spath_object.is_file(): - sketch_list.append(spath_object.parent) + p_ino = Path(sk_ino) + if p_ino.is_file(): + sketch_list.append(p_ino.parent) sketch_list = sorted(set(sketch_list)) From ce15140483f3549d0933babbdd105359b81a74cd Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Tue, 19 Oct 2021 14:12:15 +0200 Subject: [PATCH 11/17] ci(arduino_cli): manage additional_urls in the config Signed-off-by: Frederic Pillon --- CI/build/arduino-cli.py | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index bb9e97410e..b1b79e63e3 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -194,6 +194,7 @@ def check_config(): else: arduino_cli = str(arduino_cli_path / "arduino-cli") + # Check arduino-cli version try: output = subprocess.check_output( [arduino_cli, "version"], @@ -217,6 +218,39 @@ def check_config(): if args.url: stm32_url = args.url + # Check if url is already part of the arduino-cli config + try: + output = subprocess.check_output( + [ + arduino_cli, + "config", + "dump", + ], + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as e: + print(f"'{' '.join(e.cmd)}' failed with code: {e.returncode}!") + print(e.stdout.decode("utf-8")) + quit(e.returncode) + else: + if stm32_url not in output.decode("utf-8"): + # Add it to the config + try: + output = subprocess.check_output( + [ + arduino_cli, + "config", + "add", + "board_manager.additional_urls", + stm32_url, + ], + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as e: + print(f"'{' '.join(e.cmd)}' failed with code: {e.returncode}!") + print(e.stdout.decode("utf-8")) + quit(e.returncode) + # Check if requested platform is installed try: output = subprocess.check_output( [ @@ -224,8 +258,6 @@ def check_config(): "core", "search", "stm32", - "--additional-urls", - stm32_url, ], stderr=subprocess.STDOUT, ) @@ -492,8 +524,6 @@ def find_board(): arduino_cli, "board", "details", - "--additional-urls", - stm32_url, "--format", "json", "-b", From 4e7689065b6494283117305df54de4a10e0530f8 Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Tue, 19 Oct 2021 16:37:06 +0200 Subject: [PATCH 12/17] ci(arduino_cli): remove useless code Signed-off-by: Frederic Pillon --- CI/build/arduino-cli.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index b1b79e63e3..6c2272e6c3 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -746,17 +746,15 @@ def build_config(sketch, boardSkipped): else: # get specific sketch options to append to the fqbn for pattern in sketch_options: - print - if pattern in sketch_options: - if re.search(pattern, str(sketch), re.IGNORECASE): - if build_conf_list[idx][4][-2].count(":") == 3: - build_conf_list[idx][4][-2] += ( - "," + sketch_options[pattern] - ) - else: - build_conf_list[idx][4][-2] += ( - ":" + sketch_options[pattern] - ) + if re.search(pattern, str(sketch), re.IGNORECASE): + if build_conf_list[idx][4][-2].count(":") == 3: + build_conf_list[idx][4][-2] += ( + "," + sketch_options[pattern] + ) + else: + build_conf_list[idx][4][-2] += ( + ":" + sketch_options[pattern] + ) return build_conf_list From 7524f0192896d3a545278170b8a563e7185fe7f7 Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Tue, 19 Oct 2021 16:54:57 +0200 Subject: [PATCH 13/17] ci(arduino_cli): remove skipped from the log and stats Only display the number of skipped build. Signed-off-by: Frederic Pillon --- CI/build/arduino-cli.py | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index 6c2272e6c3..b5404941b0 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -640,10 +640,9 @@ def log_sketch_build_result(sketch, boardKo, boardSkipped): # Log final result def log_final_result(): # Also equal to len(board_fqbn) * len(sketch_list) - nb_build_total = nb_build_passed + nb_build_failed + nb_build_skipped + nb_build_total = nb_build_passed + nb_build_failed stat_passed = round(nb_build_passed * 100.0 / nb_build_total, 2) stat_failed = round(nb_build_failed * 100.0 / nb_build_total, 2) - stat_skipped = round(nb_build_skipped * 100.0 / nb_build_total, 2) duration = str(timedelta(seconds=time.time() - full_buildTime)) # Log file @@ -653,8 +652,8 @@ def log_final_result(): f.write(build_separator + "\n") ssucc = f"{nb_build_passed} succeeded ({stat_passed}%)" sfail = f"{nb_build_failed} failed ({stat_failed}%)" - sskip = f"{nb_build_skipped} skipped ({stat_skipped}%)" - f.write(f"{ssucc}, {sfail}, {sskip} of {nb_build_total} builds\n") + sskip = f"{nb_build_skipped} skipped)" + f.write(f"{ssucc}, {sfail} of {nb_build_total} builds ({sskip})\n") f.write(f"Ends {time.strftime('%A %d %B %Y %H:%M:%S')}\n") f.write(f"Duration: {duration}\n") f.write(f"Logs are available here:\n{output_dir}\n") @@ -663,8 +662,8 @@ def log_final_result(): # Standard output ssucc = f"{nb_build_passed} {fsucc} ({stat_passed}%)" sfail = f"{nb_build_failed} {ffail} ({stat_failed}%)" - sskip = f"{nb_build_skipped} {fskip} ({stat_skipped}%)" - print(f"Builds Summary: {ssucc}, {sfail}, {sskip} of {nb_build_total} builds") + sskip = f"{nb_build_skipped} {fskip}" + print(f"Builds Summary: {ssucc}, {sfail} of {nb_build_total} builds ({sskip})") print(f"Duration: {duration}") print("Logs are available here:") print(output_dir) @@ -676,7 +675,7 @@ def get_fqbn(b_name): return board_custom_fqbn[b_name] else: if b_name in board_options and board_options[b_name]: - return board_fqbn[b_name] + "," + board_options[b_name] + return f"{board_fqbn[b_name]},{board_options[b_name]}" else: return board_fqbn[b_name] @@ -728,17 +727,6 @@ def build_config(sketch, boardSkipped): if build_conf_list[idx][0] in na_sketch_pattern: for pattern in na_sketch_pattern[build_conf_list[idx][0]]: if re.search(pattern, str(sketch), re.IGNORECASE): - print( - (build_format_result).format( - "{}/{}".format( - build_conf_list[idx][1], build_conf_list[idx][2] - ), - build_conf_list[idx][0], - fskip, - 0.00, - ) - ) - boardSkipped.append(build_conf_list[idx][0]) del build_conf_list[idx] nb_build_skipped += 1 @@ -771,8 +759,7 @@ def build_all(): for sketch_nb, sketch in enumerate(sketch_list, start=1): boardKo = [] boardSkipped = [] - print("\n") - print(build_separator) + print(f"\n{build_separator}") print( "| {:^85} |".format( f"Sketch \033[34m{sketch.name}\033[0m ({sketch_nb}/{len(sketch_list)})" From 927db15a9d8f7fcd188cf6d55574831404e5e092 Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Tue, 19 Oct 2021 18:39:39 +0200 Subject: [PATCH 14/17] ci(arduino_cli): rework build index to ignore skipped Signed-off-by: Frederic Pillon --- CI/build/arduino-cli.py | 88 ++++++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 41 deletions(-) diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index b5404941b0..39814397c8 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -105,6 +105,12 @@ fskip = "\033[33mskipped\033[0m" nl = "\n" +# index build configuration +idx_b_name = 0 +idx_build = 1 +idx_log = 2 +idx_cmd = 3 + def cat(file): with open(file, "r") as f: @@ -557,17 +563,17 @@ def find_board(): # Check the status -def check_status(status, build_conf, boardKo): +def check_status(status, build_conf, boardKo, nb_build_conf): global nb_build_passed global nb_build_failed - sketch_name = build_conf[4][-1].name + sketch_name = build_conf[idx_cmd][-1].name if status[1] == 0: result = fsucc nb_build_passed += 1 elif status[1] == 1: # Check if failed due to a region overflowed - logFile = build_conf[3] / f"{sketch_name}.log" + logFile = build_conf[idx_log] / f"{sketch_name}.log" error_found = False for i, line in enumerate(open(logFile)): if error_pattern.search(line): @@ -580,7 +586,7 @@ def check_status(status, build_conf, boardKo): error_found = True if error_found: result = ffail - boardKo.append(build_conf[0]) + boardKo.append(build_conf[idx_b_name]) if args.ci: cat(logFile) nb_build_failed += 1 @@ -591,12 +597,12 @@ def check_status(status, build_conf, boardKo): nb_build_passed += 1 else: result = "\033[31merror\033[0m" - boardKo.append(build_conf[0]) + boardKo.append(build_conf[idx_b_name]) nb_build_failed += 1 print( f"{build_format_result}".format( - f"{build_conf[1]}/{build_conf[2]}", - build_conf[0], + f"{build_conf[idx_build]}/{nb_build_conf}", + build_conf[idx_b_name], result, status[0], ) @@ -702,47 +708,46 @@ def genBasicCommand(b_name): def create_build_conf_list(): build_conf_list = [] - idx = 1 - for b_name in board_fqbn: + for idx, b_name in enumerate(board_fqbn, start=1): build_conf_list.append( - ( + [ b_name, idx, - len(board_fqbn), output_dir / b_name, genBasicCommand(b_name), - ) + ] ) - idx += 1 return build_conf_list def build_config(sketch, boardSkipped): global nb_build_skipped - build_conf_list = create_build_conf_list() - - for idx in reversed(range(len(build_conf_list))): - build_conf_list[idx][4][-1] = sketch - if na_sketch_pattern: - if build_conf_list[idx][0] in na_sketch_pattern: - for pattern in na_sketch_pattern[build_conf_list[idx][0]]: - if re.search(pattern, str(sketch), re.IGNORECASE): - boardSkipped.append(build_conf_list[idx][0]) - del build_conf_list[idx] - nb_build_skipped += 1 - break - else: - # get specific sketch options to append to the fqbn - for pattern in sketch_options: - if re.search(pattern, str(sketch), re.IGNORECASE): - if build_conf_list[idx][4][-2].count(":") == 3: - build_conf_list[idx][4][-2] += ( - "," + sketch_options[pattern] - ) - else: - build_conf_list[idx][4][-2] += ( - ":" + sketch_options[pattern] - ) + build_conf_list = [] + build_conf_list_tmp = create_build_conf_list() + + while len(build_conf_list_tmp): + build_conf = build_conf_list_tmp.pop(0) + build_conf[idx_cmd][-1] = sketch + b_name = build_conf[idx_b_name] + s_sketch = str(sketch) + build_conf[idx_build] = len(build_conf_list) + 1 + if b_name in na_sketch_pattern: + for pattern in na_sketch_pattern[b_name]: + if re.search(pattern, s_sketch, re.IGNORECASE): + boardSkipped.append(b_name) + nb_build_skipped += 1 + break + else: + # Get specific sketch options to append to the fqbn + for pattern in sketch_options: + if re.search(pattern, s_sketch, re.IGNORECASE): + if build_conf[idx_cmd][-2].count(":") == 3: + build_conf[idx_cmd][-2] += f",{sketch_options[pattern]}" + else: + build_conf[idx_cmd][-2] += f":{sketch_options[pattern]}" + build_conf_list.append(build_conf) + else: + build_conf_list.append(build_conf) return build_conf_list @@ -773,11 +778,12 @@ def build_all(): print(build_separator) build_conf_list = build_config(sketch, boardSkipped) + nb_build_conf = len(build_conf_list) with concurrent.futures.ProcessPoolExecutor(os.cpu_count() - 1) as executor: for build_conf, res in zip( build_conf_list, executor.map(build, build_conf_list) ): - check_status(res, build_conf, boardKo) + check_status(res, build_conf, boardKo, nb_build_conf) log_sketch_build_result(sketch, boardKo, boardSkipped) # Ensure no cache issue deleteFolder(build_output_cache_dir) @@ -786,9 +792,9 @@ def build_all(): # Run arduino-cli command def real_build(build_conf): - cmd = build_conf[4] + cmd = build_conf[idx_cmd] status = [time.monotonic()] - with open(build_conf[3] / f"{cmd[-1].name}.log", "w") as stdout: + with open(build_conf[idx_log] / f"{cmd[-1].name}.log", "w") as stdout: res = subprocess.Popen(cmd, stdout=stdout, stderr=subprocess.STDOUT) res.wait() status[0] = time.monotonic() - status[0] @@ -802,7 +808,7 @@ def dry_build(build_conf): status = [random.random() * 10, random.randint(0, 1)] if status[1] == 1: # Create dummy log file - logFile = build_conf[3] / f"{ build_conf[4][-1].name}.log" + logFile = build_conf[idx_log] / f"{ build_conf[idx_cmd][-1].name}.log" # random failed dummy = open(logFile, "w") if random.randint(0, 1) == 1: From 55140d841859cb159d8105020ec43c1542c7c5db Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Tue, 19 Oct 2021 18:49:04 +0200 Subject: [PATCH 15/17] ci(arduino_cli): replace arduino builder by arduino-cli Signed-off-by: Frederic Pillon --- CI/build/arduino-cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index 39814397c8..a6ba736330 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -675,7 +675,7 @@ def log_final_result(): print(output_dir) -# Set up specific options to customise arduino builder command +# Set up specific options to customise arduino-cli command def get_fqbn(b_name): if b_name in board_custom_fqbn and board_custom_fqbn[b_name]: return board_custom_fqbn[b_name] @@ -686,7 +686,7 @@ def get_fqbn(b_name): return board_fqbn[b_name] -# Generate arduino builder basic command +# Generate arduino-cli basic command def genBasicCommand(b_name): cmd = [] cmd.append(arduino_cli) From 150acb85e4387e5498dafdb65472ab89761472bb Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Thu, 21 Oct 2021 08:53:11 +0200 Subject: [PATCH 16/17] ci: remove old travis path config useless since CI moved to GitHub action. Signed-off-by: Frederic Pillon --- CI/build/conf/path_config_travis.json | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 CI/build/conf/path_config_travis.json diff --git a/CI/build/conf/path_config_travis.json b/CI/build/conf/path_config_travis.json deleted file mode 100644 index 597663e91d..0000000000 --- a/CI/build/conf/path_config_travis.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "ARDUINO_PATH": "/home/travis/IDE/arduino", - "ARDUINO_PACKAGES": "/home/travis/.arduino15/packages", - "ARDUINO_USER_SKETCHBOOK": "/home/travis/Arduino", - "BUILD_OUPUT_DIR": "/tmp/BuildOutput", - "ROOT_OUPUT_DIR": "/home/travis/arduinoBuilderOutput" -} \ No newline at end of file From aaf405bf1e59ebfebf8fc87cef9c25cac32dc3c9 Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Thu, 21 Oct 2021 10:31:23 +0200 Subject: [PATCH 17/17] ci(arduino_cli): ensure configuration exists Signed-off-by: Frederic Pillon --- CI/build/arduino-cli.py | 67 +++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index a6ba736330..8063a88d94 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -224,38 +224,47 @@ def check_config(): if args.url: stm32_url = args.url - # Check if url is already part of the arduino-cli config + # Ensure a configuration exists try: + print("Check/update arduino-cli configuration") + # Try to create configuration file output = subprocess.check_output( - [ - arduino_cli, - "config", - "dump", - ], + [arduino_cli, "config", "init", "--additional-urls", stm32_url], stderr=subprocess.STDOUT, - ) - except subprocess.CalledProcessError as e: - print(f"'{' '.join(e.cmd)}' failed with code: {e.returncode}!") - print(e.stdout.decode("utf-8")) - quit(e.returncode) - else: - if stm32_url not in output.decode("utf-8"): - # Add it to the config - try: - output = subprocess.check_output( - [ - arduino_cli, - "config", - "add", - "board_manager.additional_urls", - stm32_url, - ], - stderr=subprocess.STDOUT, - ) - except subprocess.CalledProcessError as e: - print(f"'{' '.join(e.cmd)}' failed with code: {e.returncode}!") - print(e.stdout.decode("utf-8")) - quit(e.returncode) + ).decode("utf-8") + except subprocess.CalledProcessError: + try: + output = subprocess.check_output( + [ + arduino_cli, + "config", + "dump", + ], + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as e: + print(f"'{' '.join(e.cmd)}' failed with code: {e.returncode}!") + print(e.stdout.decode("utf-8")) + quit(e.returncode) + else: + # Check if url is already part of the arduino-cli config + if stm32_url not in output.decode("utf-8"): + # Add it to the config + try: + output = subprocess.check_output( + [ + arduino_cli, + "config", + "add", + "board_manager.additional_urls", + stm32_url, + ], + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as e: + print(f"'{' '.join(e.cmd)}' failed with code: {e.returncode}!") + print(e.stdout.decode("utf-8")) + quit(e.returncode) # Check if requested platform is installed try: output = subprocess.check_output(