[CI][Docker][Cortex-M]Update scripts to update ci_cortexm to Ubuntu 20.04 (#13736)

mehrdadh · web-flow · commit a9c6f137d8eb · 2023-01-11T16:51:45.000Z
- Updates nrfjprog to latest (10.18.1) to fix issues related to running microTVM tests inside docker. The main issue was that with older version commands like nrfjprog --com was not working even with docker --privileged.
- Update python install script to support python 3.7 in 20.04 -&gt; we keep using python 3.7 in ubuntu 20.04 until its EOL
- Add script to install LLVM in ubuntu 20.04
diff --git a/apps/microtvm/reference-vm/base-box/base_box_setup_common.sh b/apps/microtvm/reference-vm/base-box/base_box_setup_common.sh
@@ -46,7 +46,7 @@ sudo apt-install-and-clear -y --no-install-recommends \
      cmake=3.22.2-0kitware1ubuntu18.04.1 cmake-data=3.22.2-0kitware1ubuntu18.04.1 \
 
 # Python
-sudo ~/ubuntu_install_python.sh
+sudo ~/ubuntu_install_python.sh 3.7
 rm -f ~/ubuntu_install_python.sh
 
 # Poetry deps
diff --git a/docker/Dockerfile.ci_arm b/docker/Dockerfile.ci_arm
@@ -56,7 +56,7 @@ RUN bash /install/ubuntu_install_llvm.sh
 ENV TVM_VENV /venv/apache-tvm-py3.7
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh 3.7
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
diff --git a/docker/Dockerfile.ci_cortexm b/docker/Dockerfile.ci_cortexm
@@ -17,7 +17,7 @@
 
 # CI docker CPU env
 # tag: v0.62
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
 
@@ -38,15 +38,15 @@ RUN bash /install/ubuntu_install_googletest.sh
 ENV TVM_VENV /venv/apache-tvm-py3.7
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh 3.7
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
 COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
 RUN bash /install/ubuntu_install_python_package.sh
 
-COPY install/ubuntu1804_install_llvm.sh /install/ubuntu1804_install_llvm.sh
-RUN bash /install/ubuntu1804_install_llvm.sh
+COPY install/ubuntu2004_install_llvm.sh /install/ubuntu2004_install_llvm.sh
+RUN bash /install/ubuntu2004_install_llvm.sh
 
 # Rust env (build early; takes a while)
 COPY install/ubuntu_install_rust.sh /install/ubuntu_install_rust.sh
diff --git a/docker/Dockerfile.ci_cpu b/docker/Dockerfile.ci_cpu
@@ -37,7 +37,7 @@ RUN bash /install/ubuntu_install_googletest.sh
 ENV TVM_VENV /venv/apache-tvm-py3.7
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh 3.7
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
diff --git a/docker/Dockerfile.ci_gpu b/docker/Dockerfile.ci_gpu
@@ -44,7 +44,7 @@ RUN bash /install/ubuntu_install_googletest.sh /googletest
 ENV TVM_VENV /venv/apache-tvm-py3.7
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh 3.7
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
diff --git a/docker/Dockerfile.ci_hexagon b/docker/Dockerfile.ci_hexagon
@@ -40,7 +40,7 @@ RUN bash /install/ubuntu_install_googletest.sh
 ENV TVM_VENV /venv/apache-tvm-py3.8
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh 3.8
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
diff --git a/docker/Dockerfile.ci_i386 b/docker/Dockerfile.ci_i386
@@ -52,7 +52,7 @@ ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 ENV TVM_VENV /venv/apache-tvm-py3.7
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh 3.7
 ENV PATH ${TVM_VENV}/bin:$PATH
 
 COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
diff --git a/docker/Dockerfile.ci_lint b/docker/Dockerfile.ci_lint
@@ -29,7 +29,7 @@ RUN apt-install-and-clear -y wget git sudo make parallel
 ENV TVM_VENV /venv/apache-tvm-py3.7
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh 3.7
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
diff --git a/docker/Dockerfile.ci_minimal b/docker/Dockerfile.ci_minimal
@@ -37,7 +37,7 @@ RUN bash /install/ubuntu_install_googletest.sh
 ENV TVM_VENV /venv/apache-tvm-py3.7
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh 3.7
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
diff --git a/docker/Dockerfile.ci_riscv b/docker/Dockerfile.ci_riscv
@@ -38,7 +38,7 @@ RUN bash /install/ubuntu_install_googletest.sh
 ENV TVM_VENV /venv/apache-tvm-py3.7
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh 3.7
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
diff --git a/docker/Dockerfile.ci_wasm b/docker/Dockerfile.ci_wasm
@@ -35,7 +35,7 @@ RUN bash /install/ubuntu_install_googletest.sh
 ENV TVM_VENV /venv/apache-tvm-py3.7
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh 3.7
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
diff --git a/docker/Dockerfile.demo_android b/docker/Dockerfile.demo_android
@@ -31,7 +31,7 @@ RUN bash /install/ubuntu_install_core.sh
 ENV TVM_VENV /venv/apache-tvm-py3.7
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu1804_install_python.sh
-RUN bash /install/ubuntu1804_install_python.sh
+RUN bash /install/ubuntu1804_install_python.sh 3.7
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
diff --git a/docker/Dockerfile.demo_rocm b/docker/Dockerfile.demo_rocm
@@ -29,7 +29,7 @@ RUN bash /install/ubuntu_install_core.sh
 ENV TVM_VENV /venv/apache-tvm-py3.7
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh 3.7
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
diff --git a/docker/Dockerfile.demo_vitis_ai b/docker/Dockerfile.demo_vitis_ai
@@ -35,7 +35,7 @@ RUN bash /install/ubuntu_install_vitis_ai_core.sh
 ENV TVM_VENV /venv/apache-tvm-py3.7
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh 3.7
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
diff --git a/docker/install/ubuntu2004_install_llvm.sh b/docker/install/ubuntu2004_install_llvm.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euxo pipefail
+
+echo deb http://apt.llvm.org/focal/ llvm-toolchain-focal main\
+    >> /etc/apt/sources.list.d/llvm.list
+
+echo deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main\
+    >> /etc/apt/sources.list.d/llvm.list
+
+apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 15CF4D18AF4F7421
+
+apt-get update && apt-install-and-clear -y \
+     llvm-9 llvm-10 llvm-11 llvm-12 llvm-13 \
+     clang-9 libclang-9-dev \
+     clang-10 libclang-10-dev \
+     clang-11 libclang-11-dev \
+     clang-12 libclang-12-dev \
+     clang-13 libclang-13-dev
diff --git a/docker/install/ubuntu_install_nrfjprog.sh b/docker/install/ubuntu_install_nrfjprog.sh
@@ -21,19 +21,29 @@ set -u
 set -o pipefail
 set -x
 
+TEMP_PATH_NAME="nrfjprog_tmp_path"
+
+# Install JLink
 NRF_COMMANDLINE_TOOLS_FILE=nRFCommandLineToolsLinuxamd64.tar.gz
-NRF_COMMANDLINE_TOOLS_URL=https://nsscprodmedia.blob.core.windows.net/prod/software-and-other-downloads/desktop-software/nrf-command-line-tools/sw/versions-10-x-x/10-12-1/nrfcommandlinetools10121linuxamd64.tar.gz
-NRF_COMMANDLINE_TOOLS_INSTALLER=nRF-Command-Line-Tools_10_12_1_Linux-amd64.deb
-JLINK_LINUX_INSTALLER=JLink_Linux_V688a_x86_64.deb
+NRF_COMMANDLINE_TOOLS_URL=https://nsscprodmedia.blob.core.windows.net/prod/software-and-other-downloads/desktop-software/nrf-command-line-tools/sw/versions-10-x-x/10-18-1/nrf-command-line-tools-10.18.1_linux-amd64.tar.gz
+JLINK_LINUX_INSTALLER=JLink_Linux_V780c_x86_64.deb
+NRF_COMMANDLINE_TOOLS_SHA=5611536ca3377d64131ccd51232f9e33cde6d289b03ea33db0581a1288be8b0b10f995e2d60fdd4a3ce5a5c7b12bc85ddc672b282c9af8c5808707ab41543a7d
 
 cd ~
-mkdir -p nrfjprog
+mkdir -p ${TEMP_PATH_NAME}
 wget --no-verbose -O $NRF_COMMANDLINE_TOOLS_FILE $NRF_COMMANDLINE_TOOLS_URL
+echo "$NRF_COMMANDLINE_TOOLS_SHA $NRF_COMMANDLINE_TOOLS_FILE" | sha512sum --check
 
-cd nrfjprog
+cd ${TEMP_PATH_NAME}
 tar -xzvf "../${NRF_COMMANDLINE_TOOLS_FILE}"
 apt-install-and-clear -y "./${JLINK_LINUX_INSTALLER}"
-apt-install-and-clear -y "./${NRF_COMMANDLINE_TOOLS_INSTALLER}"
+
+# Install nrfjprog
+NRF_DEB_FILE=nrf-command-line-tools_amd64.deb
+NRF_DEB_FILE_SHA=1f0339e16d50345ddde9757c2a4211361bcc78ff7371aac09decfffa809d86329001f5bc135f33dd154000a8f0da8bee4a0e80d3865ceff229f63ff9ace5ea95
+wget --no-verbose -O $NRF_DEB_FILE https://nsscprodmedia.blob.core.windows.net/prod/software-and-other-downloads/desktop-software/nrf-command-line-tools/sw/versions-10-x-x/10-18-1/nrf-command-line-tools_10.18.1_amd64.deb
+echo "$NRF_DEB_FILE_SHA $NRF_DEB_FILE" | sha512sum --check
+apt-install-and-clear -y ./$NRF_DEB_FILE
 
 cd ..
-rm -rf nrfjprog "${NRF_COMMANDLINE_TOOLS_FILE}"
+rm -rf ${TEMP_PATH_NAME} "${NRF_COMMANDLINE_TOOLS_FILE}"
diff --git a/docker/install/ubuntu_install_python.sh b/docker/install/ubuntu_install_python.sh
@@ -27,25 +27,42 @@ if [ -z "${TVM_VENV+x}" ]; then
     exit 2
 fi
 
+if [ "$#" -lt 1 ]; then
+    echo "Usage: docker/install/ubuntu_install_python.sh <PYTHON_VERSION>"
+    exit -1
+fi
+PYTHON_VERSION=$1
+
+if [ "${PYTHON_VERSION}" != "3.7" ] && [ "${PYTHON_VERSION}" != "3.8" ]; then
+    echo "Only 3.7 and 3.8 versions are supported in this script."
+    exit -1
+fi
+
 apt-get update
 
 # Ensure lsb-release is installed.
 apt-install-and-clear -y \
     lsb-core
 
+apt-install-and-clear -y software-properties-common
+
 release=$(lsb_release -sc)
 if [ "${release}" == "bionic" ]; then
-    PYTHON_VERSION=3.7
+    if [ "${PYTHON_VERSION}" == "3.8" ]; then
+        echo "Only 3.7 is supported for bionic in this script."
+        exit -1
+    fi
 elif [ "${release}" == "focal" ]; then
-    PYTHON_VERSION=3.8
+    if [ "${PYTHON_VERSION}" == "3.7" ]; then
+        add-apt-repository -y ppa:deadsnakes/ppa
+    fi
 else
     echo "Don't know which version of python to install for lsb-release ${release}"
     exit 2
 fi
 
 # Install python and pip. Don't modify this to add Python package dependencies,
 # instead modify install_python_package.sh
-apt-install-and-clear -y software-properties-common
 apt-install-and-clear -y \
     acl \
     python${PYTHON_VERSION} \
