From dd2c66372048ff320dc610deccbdc229d6b0a0db Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Sat, 18 Dec 2021 19:32:52 -0800 Subject: [PATCH 1/5] Add wheel to installed pip packages on aarch64, so we can build the wheel in the CD pipeline. --- ci/docker/install/requirements_aarch64 | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/docker/install/requirements_aarch64 b/ci/docker/install/requirements_aarch64 index 327a78f7d960..1cfead0b41fc 100644 --- a/ci/docker/install/requirements_aarch64 +++ b/ci/docker/install/requirements_aarch64 @@ -30,3 +30,4 @@ astroid==2.3.3 # pylint and astroid need to be aligned requests<2.19.0,>=2.18.4 setuptools coverage +wheel From ee400892bbcc9ab1c62ef843306d4a9e2f148271 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Sat, 18 Dec 2021 19:40:00 -0800 Subject: [PATCH 2/5] Better support for aarch64 docker container cache. --- ci/build.py | 11 ++++++----- ci/docker_cache.py | 3 ++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/ci/build.py b/ci/build.py index b7236dcc9beb..a6e18485625b 100755 --- a/ci/build.py +++ b/ci/build.py @@ -34,6 +34,7 @@ import shutil import signal import subprocess +import platform from itertools import chain from subprocess import check_call, check_output from typing import * @@ -60,7 +61,7 @@ def get_docker_compose_platforms(path: str = get_dockerfiles_path()): return platforms -def get_platforms(path: str = get_dockerfiles_path(), arch='x86') -> List[str]: +def get_platforms(path: str = get_dockerfiles_path(), arch='x86_64') -> List[str]: """Get a list of platforms given our dockerfiles""" dockerfiles = glob.glob(os.path.join(path, "Dockerfile.*")) dockerfiles = set(filter(lambda x: x[-1] != '~', dockerfiles)) @@ -68,7 +69,7 @@ def get_platforms(path: str = get_dockerfiles_path(), arch='x86') -> List[str]: files = files - DOCKER_COMPOSE_FILES files.update(["build."+x for x in get_docker_compose_platforms()]) arm_files = set(filter(lambda x: any(y in x for y in AARCH_FILE_KEYWORDS), files)) - if arch == 'x86': + if arch == 'x86_64': files = files - arm_files elif arch == 'aarch64': files = arm_files @@ -300,7 +301,7 @@ def docker_run_cmd(cmd): return 0 -def list_platforms(arch='x86') -> str: +def list_platforms(arch='x86_64') -> str: return "\nSupported platforms:\n{}".format('\n'.join(get_platforms(arch=arch))) @@ -357,8 +358,8 @@ def main() -> int: type=str) parser.add_argument("-A", "--architecture", - help="Architecture of images to build (x86 or aarch64). Default is x86.", - default='x86', + help="Architecture of images to build (x86_64 or aarch64). Default is current machine type.", + default=platform.machine(), dest='architecture') parser.add_argument("-b", "--build-only", diff --git a/ci/docker_cache.py b/ci/docker_cache.py index f9d5b81dff2b..d320bf9ef41f 100755 --- a/ci/docker_cache.py +++ b/ci/docker_cache.py @@ -30,6 +30,7 @@ import subprocess import re import sys +import platform from typing import * import build as build_util @@ -225,7 +226,7 @@ def script_name() -> str: args = parser.parse_args() - platforms = build_util.get_platforms() + platforms = build_util.get_platforms(arch=platform.machine()) if "dkr.ecr" in args.docker_registry: _ecr_login(args.docker_registry) From 2c243bd7a98c291925106266302b93330d817988 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Sat, 18 Dec 2021 19:47:16 -0800 Subject: [PATCH 3/5] Only import machine() function from platform module, as platform variable is already widely used. --- ci/build.py | 4 ++-- ci/docker_cache.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/build.py b/ci/build.py index a6e18485625b..18f7a81a4a40 100755 --- a/ci/build.py +++ b/ci/build.py @@ -34,7 +34,7 @@ import shutil import signal import subprocess -import platform +from platform import machine from itertools import chain from subprocess import check_call, check_output from typing import * @@ -359,7 +359,7 @@ def main() -> int: parser.add_argument("-A", "--architecture", help="Architecture of images to build (x86_64 or aarch64). Default is current machine type.", - default=platform.machine(), + default=machine(), dest='architecture') parser.add_argument("-b", "--build-only", diff --git a/ci/docker_cache.py b/ci/docker_cache.py index d320bf9ef41f..a51203480d7e 100755 --- a/ci/docker_cache.py +++ b/ci/docker_cache.py @@ -30,7 +30,7 @@ import subprocess import re import sys -import platform +from platform import machine from typing import * import build as build_util @@ -226,7 +226,7 @@ def script_name() -> str: args = parser.parse_args() - platforms = build_util.get_platforms(arch=platform.machine()) + platforms = build_util.get_platforms(arch=machine()) if "dkr.ecr" in args.docker_registry: _ecr_login(args.docker_registry) From b239cff564661f01dbd8f894636f2457061adcdf Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Sat, 18 Dec 2021 19:51:55 -0800 Subject: [PATCH 4/5] Only build containers with aarch64 in name on aarch64. --- ci/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build.py b/ci/build.py index 18f7a81a4a40..dc36a506f274 100755 --- a/ci/build.py +++ b/ci/build.py @@ -47,7 +47,7 @@ DOCKER_COMPOSE_FILES = set(['docker/build.centos7']) # keywords to identify arm-based dockerfiles -AARCH_FILE_KEYWORDS = ['armv', 'aarch64'] +AARCH_FILE_KEYWORDS = ['aarch64'] def get_dockerfiles_path(): return "docker" From 788e49a1dfc53f75ec56ad3b1754821fd227716e Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Sat, 18 Dec 2021 20:01:26 -0800 Subject: [PATCH 5/5] Make all arch defaults to local machine type. --- ci/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build.py b/ci/build.py index dc36a506f274..1998f58ee272 100755 --- a/ci/build.py +++ b/ci/build.py @@ -61,7 +61,7 @@ def get_docker_compose_platforms(path: str = get_dockerfiles_path()): return platforms -def get_platforms(path: str = get_dockerfiles_path(), arch='x86_64') -> List[str]: +def get_platforms(path: str = get_dockerfiles_path(), arch=machine()) -> List[str]: """Get a list of platforms given our dockerfiles""" dockerfiles = glob.glob(os.path.join(path, "Dockerfile.*")) dockerfiles = set(filter(lambda x: x[-1] != '~', dockerfiles)) @@ -301,7 +301,7 @@ def docker_run_cmd(cmd): return 0 -def list_platforms(arch='x86_64') -> str: +def list_platforms(arch=machine()) -> str: return "\nSupported platforms:\n{}".format('\n'.join(get_platforms(arch=arch)))