CI: Update Android, ARM build and ARM test containers (apache#18264)

* Android build containers

* ARM build containers

* ARM test containers

* Fix naming scheme

* Set WORKDIR at correct location

ci/build.py

@@ -46,9 +46,11 @@
 DOCKER_COMPOSE_WHITELIST = ('centos7_cpu', 'centos7_gpu_cu92', 'centos7_gpu_cu100',
                             'centos7_gpu_cu101', 'centos7_gpu_cu102', 'ubuntu_cpu',
                             'ubuntu_build_cuda', 'ubuntu_gpu_cu101', 'publish.test.centos7_cpu',
-                            'publish.test.centos7_gpu')
+                            'publish.test.centos7_gpu', 'android_armv7', 'android_armv8',
+                            'armv6', 'armv7', 'armv8', 'test.armv7', 'test.armv8')
 # Files for docker compose
-DOCKER_COMPOSE_FILES = set(('docker/build.centos7', 'docker/build.ubuntu', 'docker/publish.test.centos7'))
+DOCKER_COMPOSE_FILES = set(('docker/build.centos7', 'docker/build.ubuntu', 'docker/build.android',
+                            'docker/build.arm', 'docker/test.arm', 'docker/publish.test.centos7'))
 
 
 def get_dockerfiles_path():

ci/docker/Dockerfile.build.android_armv7 → ci/docker/Dockerfile.build.android

@@ -16,14 +16,12 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build MXNet for Android ARMv7
+# Dockerfile to build MXNet for Android
 
-FROM ubuntu:20.04
-
-ENV ARCH=armv7l \
-    HOSTCC=gcc \
-    HOSTCXX=g++ \
-    TARGET=ARMV7
+####################################################################################################
+# Shared base for all Android targets
+####################################################################################################
+FROM ubuntu:20.04 AS base
 
 WORKDIR /usr/local
 
@@ -42,6 +40,23 @@ RUN curl -o android-ndk-r19-linux-x86_64.zip -L https://dl.google.com/android/re
     rm android-ndk-r19-linux-x86_64.zip
 ENV CMAKE_TOOLCHAIN_FILE=/usr/local/android-ndk-r19/build/cmake/android.toolchain.cmake
 
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY runtime_functions.sh /work/
+
+
+####################################################################################################
+# Specialize base image for ARMv7
+####################################################################################################
+FROM base as armv7
+ENV ARCH=armv7l \
+    HOSTCC=gcc \
+    HOSTCXX=g++ \
+    TARGET=ARMV7
+
 RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \
     mkdir /usr/local/openblas-android && \
     cd /usr/local/OpenBLAS && \
@@ -54,10 +69,28 @@ RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \
     rm -rf OpenBLAS
 ENV OpenBLAS_HOME=/usr/local/openblas-android
 
-ARG USER_ID=0
-ARG GROUP_ID=0
-COPY install/ubuntu_adduser.sh /work/
-RUN /work/ubuntu_adduser.sh
+WORKDIR /work/build
+
+
+####################################################################################################
+# Specialize base image for ARMv8
+####################################################################################################
+FROM base as armv8
+ENV ARCH=aarch64 \
+    HOSTCC=gcc \
+    HOSTCXX=g++ \
+    TARGET=ARMV8
+
+RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \
+    mkdir /usr/local/openblas-android && \
+    cd /usr/local/OpenBLAS && \
+    export TOOLCHAIN=/usr/local/android-ndk-r19/toolchains/llvm/prebuilt/linux-x86_64 && \
+    make NOFORTRAN=1 NO_SHARED=1 \
+        LDFLAGS="-L/usr/local/android-ndk-r21/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/lib/gcc/aarch64-linux-android/4.9.x -lm" \
+        CC=$TOOLCHAIN/bin/aarch64-linux-android21-clang AR=$TOOLCHAIN/bin/aarch64-linux-android-ar && \
+    make PREFIX=/usr/local/openblas-android NO_SHARED=1 install && \
+    cd /usr/local && \
+    rm -rf OpenBLAS
+ENV OpenBLAS_HOME=/usr/local/openblas-android
 
-COPY runtime_functions.sh /work/
 WORKDIR /work/build

ci/docker/Dockerfile.build.armv6 → ci/docker/Dockerfile.build.arm

@@ -16,14 +16,12 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build MXNet for ARMv6
+# Dockerfile to build MXNet for ARM
 
-FROM ubuntu:20.04
-
-ENV ARCH=armv6l \
-    HOSTCC=gcc \
-    HOSTCXX=g++ \
-    TARGET=ARMV6
+####################################################################################################
+# Shared base for all ARM targets
+####################################################################################################
+FROM ubuntu:20.04 AS base
 
 WORKDIR /usr/local
 
@@ -39,6 +37,24 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     python3-pip \
  && rm -rf /var/lib/apt/lists/*
 
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY runtime_functions.sh /work/
+
+
+####################################################################################################
+# Specialize base image for ARMv6
+####################################################################################################
+FROM base as armv6
+
+ENV ARCH=armv6l \
+    HOSTCC=gcc \
+    HOSTCXX=g++ \
+    TARGET=ARMV6
+
 # We use a toolchain from toolchains.bootlin.com instead of Debian / Ubunut
 # crossbuild-essential-armel toolchain, as the latter targets ARM architecture
 # versions 4T, 5T, and 6, whereas we only wish to target ARMV6 and like to use
@@ -55,10 +71,58 @@ RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \
     cd /usr/local && \
     rm -rf OpenBLAS
 
-ARG USER_ID=0
-ARG GROUP_ID=0
-COPY install/ubuntu_adduser.sh /work/
-RUN /work/ubuntu_adduser.sh
+WORKDIR /work/mxnet
+
+
+####################################################################################################
+# Specialize base image for ARMv7
+####################################################################################################
+FROM base as armv7
+
+ENV ARCH=armv7l \
+    HOSTCC=gcc \
+    HOSTCXX=g++ \
+    TARGET=ARMV7
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    crossbuild-essential-armhf \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY toolchains/arm-linux-gnueabihf-toolchain.cmake /usr/local
+ENV CMAKE_TOOLCHAIN_FILE=/usr/local/arm-linux-gnueabihf-toolchain.cmake
+
+RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \
+    cd /usr/local/OpenBLAS && \
+    make NOFORTRAN=1 NO_SHARED=1 CC=arm-linux-gnueabihf-gcc && \
+    make PREFIX=/usr/local/arm-linux-gnueabihf NO_SHARED=1 install && \
+    cd /usr/local && \
+    rm -rf OpenBLAS
+
+WORKDIR /work/mxnet
+
+
+####################################################################################################
+# Specialize base image for ARMv8
+####################################################################################################
+FROM base as armv8
+
+ENV ARCH=aarch64 \
+    HOSTCC=gcc \
+    HOSTCXX=g++ \
+    TARGET=ARMV8
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    crossbuild-essential-arm64 \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY toolchains/aarch64-linux-gnu-toolchain.cmake /usr
+ENV CMAKE_TOOLCHAIN_FILE=/usr/aarch64-linux-gnu-toolchain.cmake
+
+RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \
+    cd /usr/local/OpenBLAS && \
+    make NOFORTRAN=1 NO_SHARED=1 CC=aarch64-linux-gnu-gcc && \
+    make PREFIX=/usr/aarch64-linux-gnu NO_SHARED=1 install && \
+    cd /usr/local && \
+    rm -rf OpenBLAS
 
-COPY runtime_functions.sh /work/
 WORKDIR /work/mxnet