From 7d9d512e63b3dd824a73f4ff4c7dbd8d829f0a3a Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Tue, 1 Feb 2022 13:50:36 -0800 Subject: [PATCH 1/8] Update to zephyr2.7 and Refactor --- apps/microtvm/reference-vm/base-box-tool.py | 8 ++++- .../zephyr/base-box/base_box_provision.sh | 6 ++-- .../zephyr/base-box/base_box_setup.sh | 9 ++--- docker/install/ubuntu_init_zephyr_project.sh | 18 ++++++---- docker/install/ubuntu_install_zephyr.sh | 12 ++----- docker/install/ubuntu_install_zephyr_sdk.sh | 33 +++++++++++++++++++ 6 files changed, 59 insertions(+), 27 deletions(-) create mode 100755 docker/install/ubuntu_install_zephyr_sdk.sh diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py index 26db8c4af6f1..a190c582fd3d 100755 --- a/apps/microtvm/reference-vm/base-box-tool.py +++ b/apps/microtvm/reference-vm/base-box-tool.py @@ -51,7 +51,7 @@ # in [platform]/base-box/base_box_provision.sh EXTRA_SCRIPTS = { "arduino": (), - "zephyr": ("docker/install/ubuntu_init_zephyr_project.sh",), + "zephyr": ("docker/install/ubuntu_init_zephyr_project.sh", "docker/install/ubuntu_install_zephyr_sdk.sh"), } PACKER_FILE_NAME = "packer.json" @@ -230,6 +230,12 @@ def generate_packer_config(platform, file_path, providers): filename = os.path.basename(script_path) provisioners.append({"type": "file", "source": script_path, "destination": f"~/{filename}"}) + provisioners.append( + { + "type": "shell", + "script": "base_box_setup.sh", + } + ) provisioners.append( { "type": "shell", diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh index 0e83d1b8be97..7d546631149f 100644 --- a/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh +++ b/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh @@ -27,9 +27,7 @@ source ~/.profile # Init Zephyr cd ~ -# Using most recent commit that passes all the tests. -ZEPHYR_VERSION="v2.5-branch" -~/ubuntu_init_zephyr_project.sh ~/zephyr ${ZEPHYR_VERSION} --commit dabf23758417fd041fec2a2a821d8f526afac29d +~/ubuntu_init_zephyr_project.sh ~/zephyr # Cleanup -rm -f *.sh +rm -f ubuntu_init_zephyr_project.sh diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh index e8385af9f663..5353700e852d 100644 --- a/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh +++ b/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh @@ -85,13 +85,8 @@ echo PATH=$PATH cd ~ echo "Downloading zephyr SDK..." -ZEPHYR_SDK_VERSION=0.12.3 -ZEPHYR_SDK_FILE=zephyr-sdk-linux-setup.run -wget --no-verbose -O $ZEPHYR_SDK_FILE \ - https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v${ZEPHYR_SDK_VERSION}/zephyr-sdk-${ZEPHYR_SDK_VERSION}-x86_64-linux-setup.run -chmod +x $ZEPHYR_SDK_FILE -"./$ZEPHYR_SDK_FILE" -- -d ~/zephyr-sdk -y -rm -rf "${ZEPHYR_SDK_FILE}" +~/ubuntu_install_zephyr_sdk.sh ~/zephyr-sdk +rm -f ubuntu_install_zephyr_sdk.sh # GDB for Zephyr SDK depends on python3.8 sudo add-apt-repository ppa:deadsnakes/ppa diff --git a/docker/install/ubuntu_init_zephyr_project.sh b/docker/install/ubuntu_init_zephyr_project.sh index 573ff30c38a8..787df7e5c675 100755 --- a/docker/install/ubuntu_init_zephyr_project.sh +++ b/docker/install/ubuntu_init_zephyr_project.sh @@ -29,20 +29,26 @@ set -x DOWNLOAD_DIR=$1 shift -ZEPHYR_BRANCH=$1 -shift -commit_hash= +if [ "$1" == "--zephyr-branch" ]; then + shift + ZEPHYR_BRANCH=$1 +else + ZEPHYR_BRANCH="v2.7-branch" + shift +fi + +COMMIT_HASH= if [ "$1" == "--commit" ]; then shift - commit_hash=$1 + COMMIT_HASH=$1 fi west init --mr ${ZEPHYR_BRANCH} ${DOWNLOAD_DIR} -if [ -n "$commit_hash" ]; then +if [ -n "$COMMIT_HASH" ]; then cd ${DOWNLOAD_DIR}/zephyr - git checkout ${commit_hash} + git checkout ${COMMIT_HASH} fi cd ${DOWNLOAD_DIR} diff --git a/docker/install/ubuntu_install_zephyr.sh b/docker/install/ubuntu_install_zephyr.sh index 566d3a5761e4..546577e6f342 100644 --- a/docker/install/ubuntu_install_zephyr.sh +++ b/docker/install/ubuntu_install_zephyr.sh @@ -47,10 +47,9 @@ pip3 install west # To keep in sync with the version # defined in apps/microtvm/zephyr/template_project/microtvm_api_server.py # We use `-branch` tag since it tracks the same version with extra patches for bugs. -ZEPHYR_VERSION="v2.7-branch" ZEPHYR_PROJECT_PATH=/opt/zephyrproject ZEPHYR_INIT_SCRIPT=$(find -name "ubuntu_init_zephyr_project.sh") -bash ${ZEPHYR_INIT_SCRIPT} ${ZEPHYR_PROJECT_PATH} ${ZEPHYR_VERSION} +bash ${ZEPHYR_INIT_SCRIPT} ${ZEPHYR_PROJECT_PATH} cd ${ZEPHYR_PROJECT_PATH} # As part of the build process, Zephyr needs to touch some symlinks in zephyr/misc/generated/syscalls_links (this path is relative to the @@ -69,10 +68,5 @@ chmod o+rwx zephyr/.cache #/opt/west/bin/pip3 install -r /opt/zephyrproject/zephyr/scripts/requirements.txt pip3 install -r /opt/zephyrproject/zephyr/scripts/requirements.txt -ZEPHYR_SDK_VERSION=0.13.2 -ZEPHYR_SDK_FILE=zephyr-sdk-linux-setup.run -wget --no-verbose -O $ZEPHYR_SDK_FILE \ - https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v${ZEPHYR_SDK_VERSION}/zephyr-sdk-${ZEPHYR_SDK_VERSION}-linux-x86_64-setup.run -chmod +x $ZEPHYR_SDK_FILE -"./$ZEPHYR_SDK_FILE" -- -d /opt/zephyr-sdk -rm "$ZEPHYR_SDK_FILE" +ZEPHYR_INSTALL_SDK_SCRIPT=$(find -name "ubuntu_install_zephyr_sdk.sh") +bash ${ZEPHYR_INSTALL_SDK_SCRIPT} /opt/zephyr-sdk diff --git a/docker/install/ubuntu_install_zephyr_sdk.sh b/docker/install/ubuntu_install_zephyr_sdk.sh new file mode 100755 index 000000000000..f0495e4a919c --- /dev/null +++ b/docker/install/ubuntu_install_zephyr_sdk.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -u +set -o pipefail +set -x + +INSTALLATION_PATH=$1 +shift + +ZEPHYR_SDK_VERSION=0.13.2 +ZEPHYR_SDK_FILE=zephyr-sdk-linux-setup.run +wget --no-verbose -O $ZEPHYR_SDK_FILE \ + https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v${ZEPHYR_SDK_VERSION}/zephyr-sdk-${ZEPHYR_SDK_VERSION}-linux-x86_64-setup.run +chmod +x $ZEPHYR_SDK_FILE +"./$ZEPHYR_SDK_FILE" -- -d ${INSTALLATION_PATH} +rm "$ZEPHYR_SDK_FILE" From abeb76b6910e2f2a5040e77c8e5014d5f9d2a4a8 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Tue, 1 Feb 2022 13:51:04 -0800 Subject: [PATCH 2/8] Temporary for testing --- apps/microtvm/reference-vm/zephyr/Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/microtvm/reference-vm/zephyr/Vagrantfile b/apps/microtvm/reference-vm/zephyr/Vagrantfile index 28339dc0a6d9..ca7d582ca57e 100644 --- a/apps/microtvm/reference-vm/zephyr/Vagrantfile +++ b/apps/microtvm/reference-vm/zephyr/Vagrantfile @@ -16,7 +16,7 @@ # under the License. Vagrant.configure("2") do |config| - config.vm.box = "tlcpack/microtvm-zephyr-2.5" + config.vm.box = "tlcpack/microtvm-zephyr-2.7-staging" if ENV.has_key?("TVM_RVM_NUM_CORES") num_cores = ENV["TVM_RVM_NUM_CORES"] From 7fbe222b4d5200d9c18d97523dcee5cf553b2fa7 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Tue, 1 Feb 2022 15:24:10 -0800 Subject: [PATCH 3/8] Update cmake version --- apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh index 5353700e852d..7d619b1f4555 100644 --- a/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh +++ b/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh @@ -38,7 +38,7 @@ sudo apt update # NOTE: latest cmake cannot be installed due to # https://github.com/zephyrproject-rtos/zephyr/issues/30232 sudo apt install -y --no-install-recommends git \ - cmake=3.18.4-0kitware1 cmake-data=3.18.4-0kitware1 \ + cmake=3.22.2-0kitware1ubuntu18.04.1 cmake-data=3.22.2-0kitware1ubuntu18.04.1 \ ninja-build gperf ccache dfu-util device-tree-compiler wget \ python3-dev python3-pip python3-setuptools python3-tk python3-wheel xz-utils file \ make gcc gcc-multilib g++-multilib libsdl2-dev From 5316910820d60270b498e88028a5d52e2a5dd0a9 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Tue, 1 Feb 2022 16:14:09 -0800 Subject: [PATCH 4/8] fix import path and format --- apps/microtvm/reference-vm/base-box-tool.py | 5 ++++- .../zephyr/template_project/microtvm_api_server.py | 9 ++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py index a190c582fd3d..5a22ab0d8819 100755 --- a/apps/microtvm/reference-vm/base-box-tool.py +++ b/apps/microtvm/reference-vm/base-box-tool.py @@ -51,7 +51,10 @@ # in [platform]/base-box/base_box_provision.sh EXTRA_SCRIPTS = { "arduino": (), - "zephyr": ("docker/install/ubuntu_init_zephyr_project.sh", "docker/install/ubuntu_install_zephyr_sdk.sh"), + "zephyr": ( + "docker/install/ubuntu_init_zephyr_project.sh", + "docker/install/ubuntu_install_zephyr_sdk.sh", + ), } PACKER_FILE_NAME = "packer.json" diff --git a/apps/microtvm/zephyr/template_project/microtvm_api_server.py b/apps/microtvm/zephyr/template_project/microtvm_api_server.py index 0d9ab3534e02..a7774899acb8 100644 --- a/apps/microtvm/zephyr/template_project/microtvm_api_server.py +++ b/apps/microtvm/zephyr/template_project/microtvm_api_server.py @@ -590,7 +590,14 @@ def _set_nonblock(fd): class ZephyrSerialTransport: @classmethod def _lookup_baud_rate(cls, options): - sys.path.insert(0, os.path.join(get_zephyr_base(options), "scripts", "dts")) + # TODO(mehrdadh): remove this hack once dtlib.py is a standalone project + # https://github.com/zephyrproject-rtos/zephyr/blob/v2.7-branch/scripts/dts/README.txt + sys.path.insert( + 0, + os.path.join( + get_zephyr_base(options), "scripts", "dts", "python-devicetree", "src", "devicetree" + ), + ) try: import dtlib # pylint: disable=import-outside-toplevel finally: From 3fc68f8d0c0234929131a1d734b5b811abfa2fd2 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Tue, 1 Feb 2022 16:27:14 -0800 Subject: [PATCH 5/8] Fix test script --- .../microtvm/reference-vm/zephyr/base-box/base_box_test.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh index a1da0bbe4e35..accaef50ffca 100755 --- a/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh +++ b/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh @@ -31,9 +31,4 @@ fi board=$1 pytest tests/micro/zephyr/test_zephyr.py --zephyr-board=${board} - -if [ $board == "stm32f746g_disco" ] || [ $board == "nucleo_f746zg" ]; then - echo "NOTE: skipped test_zephyr_aot.py on $board -- known failure" -else - pytest tests/micro/zephyr/test_zephyr_aot.py --zephyr-board=${board} -fi +pytest tests/micro/zephyr/test_zephyr_aot.py --zephyr-board=${board} From d639154d423b7ca911ece2933ded686b44e3f342 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Wed, 2 Feb 2022 09:02:34 -0800 Subject: [PATCH 6/8] address comments --- docker/Dockerfile.ci_qemu | 1 + docker/install/ubuntu_init_zephyr_project.sh | 49 +++++++++++++------- docker/install/ubuntu_install_zephyr.sh | 3 -- docker/install/ubuntu_install_zephyr_sdk.sh | 22 ++++++++- 4 files changed, 54 insertions(+), 21 deletions(-) diff --git a/docker/Dockerfile.ci_qemu b/docker/Dockerfile.ci_qemu index 2cae59c35d67..d4b496f8d6aa 100644 --- a/docker/Dockerfile.ci_qemu +++ b/docker/Dockerfile.ci_qemu @@ -67,6 +67,7 @@ RUN bash /install/ubuntu_install_sccache.sh # Zephyr SDK deps COPY install/ubuntu_install_zephyr.sh /install/ubuntu_install_zephyr.sh COPY install/ubuntu_init_zephyr_project.sh /install/ubuntu_init_zephyr_project.sh +COPY install/ubuntu_install_zephyr_sdk.sh /install/ubuntu_install_zephyr_sdk.sh RUN bash /install/ubuntu_install_zephyr.sh ENV ZEPHYR_BASE=/opt/zephyrproject/zephyr diff --git a/docker/install/ubuntu_init_zephyr_project.sh b/docker/install/ubuntu_init_zephyr_project.sh index 787df7e5c675..c1cdcaecceb9 100755 --- a/docker/install/ubuntu_init_zephyr_project.sh +++ b/docker/install/ubuntu_init_zephyr_project.sh @@ -19,38 +19,55 @@ # # Initialize Zephyr Project. # -# Usage: ubuntu_init_zephyr_project.sh path branch [--commit hash] -# path is the installation path for the repository. -# branch is the zephyr branch. -# --commit is the commit hash number of zephyrproject repository. If not specified, it uses the latest commit. +# Usage: docker/install/ubuntu_init_zephyr_project.sh [--branch BRANCH] +# [--commit HASH] +# INSTALLATION_PATH is the installation path for the repository. +# --branch is the zephyr branch. If not specified, it uses the default. +# --commit is the commit hash number of zephyrproject repository. If not specified, it uses the latest commit. # set -x -DOWNLOAD_DIR=$1 +function show_usage() { + cat < [--branch BRANCH] + [--commit COMMIT] +INSTALLATION_PATH is the installation path for the repository. +--branch is the zephyr branch. If not specified, it uses the default. +--commit is the commit hash number of zephyrproject repository. If not specified, it uses the latest commit. +EOF +} + +if [ "$#" -lt 1 -o "$1" == "--help" -o "$1" == "-h" ]; then + show_usage + exit -1 +fi + +INSTALLATION_PATH=$1 shift -if [ "$1" == "--zephyr-branch" ]; then +if [ "$1" == "--branch" ]; then shift - ZEPHYR_BRANCH=$1 -else - ZEPHYR_BRANCH="v2.7-branch" + BRANCH=$1 shift +else + BRANCH="v2.7-branch" fi -COMMIT_HASH= +COMMIT= if [ "$1" == "--commit" ]; then shift - COMMIT_HASH=$1 + COMMIT=$1 + shift fi -west init --mr ${ZEPHYR_BRANCH} ${DOWNLOAD_DIR} +west init --mr ${BRANCH} ${INSTALLATION_PATH} -if [ -n "$COMMIT_HASH" ]; then - cd ${DOWNLOAD_DIR}/zephyr - git checkout ${COMMIT_HASH} +if [ -n "$COMMIT" ]; then + cd ${INSTALLATION_PATH}/zephyr + git checkout ${COMMIT} fi -cd ${DOWNLOAD_DIR} +cd ${INSTALLATION_PATH} west update west zephyr-export diff --git a/docker/install/ubuntu_install_zephyr.sh b/docker/install/ubuntu_install_zephyr.sh index 546577e6f342..48c3dde7806f 100644 --- a/docker/install/ubuntu_install_zephyr.sh +++ b/docker/install/ubuntu_install_zephyr.sh @@ -44,9 +44,6 @@ sudo apt-get install -y cmake pip3 install west # Init ZephyrProject -# To keep in sync with the version -# defined in apps/microtvm/zephyr/template_project/microtvm_api_server.py -# We use `-branch` tag since it tracks the same version with extra patches for bugs. ZEPHYR_PROJECT_PATH=/opt/zephyrproject ZEPHYR_INIT_SCRIPT=$(find -name "ubuntu_init_zephyr_project.sh") bash ${ZEPHYR_INIT_SCRIPT} ${ZEPHYR_PROJECT_PATH} diff --git a/docker/install/ubuntu_install_zephyr_sdk.sh b/docker/install/ubuntu_install_zephyr_sdk.sh index f0495e4a919c..99ceec0a2956 100755 --- a/docker/install/ubuntu_install_zephyr_sdk.sh +++ b/docker/install/ubuntu_install_zephyr_sdk.sh @@ -15,12 +15,30 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# + +# +# Install Zephyr SDK +# +# Usage: docker/install/ubuntu_install_zephyr_sdk.sh +# INSTALLATION_PATH is the installation path for the SDK. +# set -e -set -u -set -o pipefail set -x +function show_usage() { + cat < +INSTALLATION_PATH is the installation path for the SDK. +EOF +} + +if [ "$#" -lt 1 -o "$1" == "--help" -o "$1" == "-h" ]; then + show_usage + exit -1 +fi + INSTALLATION_PATH=$1 shift From de3e49d94477921e8f3c62f23cf2aea5ae8f2adf Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Wed, 2 Feb 2022 11:50:35 -0800 Subject: [PATCH 7/8] fix path --- docker/install/ubuntu_install_zephyr.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docker/install/ubuntu_install_zephyr.sh b/docker/install/ubuntu_install_zephyr.sh index 48c3dde7806f..b2cf1c0ba4d3 100644 --- a/docker/install/ubuntu_install_zephyr.sh +++ b/docker/install/ubuntu_install_zephyr.sh @@ -45,8 +45,7 @@ pip3 install west # Init ZephyrProject ZEPHYR_PROJECT_PATH=/opt/zephyrproject -ZEPHYR_INIT_SCRIPT=$(find -name "ubuntu_init_zephyr_project.sh") -bash ${ZEPHYR_INIT_SCRIPT} ${ZEPHYR_PROJECT_PATH} +bash /install/ubuntu_init_zephyr_project.sh ${ZEPHYR_PROJECT_PATH} cd ${ZEPHYR_PROJECT_PATH} # As part of the build process, Zephyr needs to touch some symlinks in zephyr/misc/generated/syscalls_links (this path is relative to the @@ -65,5 +64,4 @@ chmod o+rwx zephyr/.cache #/opt/west/bin/pip3 install -r /opt/zephyrproject/zephyr/scripts/requirements.txt pip3 install -r /opt/zephyrproject/zephyr/scripts/requirements.txt -ZEPHYR_INSTALL_SDK_SCRIPT=$(find -name "ubuntu_install_zephyr_sdk.sh") -bash ${ZEPHYR_INSTALL_SDK_SCRIPT} /opt/zephyr-sdk +bash /install/ubuntu_install_zephyr_sdk.sh /opt/zephyr-sdk From 3f2a526963271d6493cc4c2bf2691e3f36ef362f Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Wed, 2 Feb 2022 16:48:28 -0800 Subject: [PATCH 8/8] fix image name --- apps/microtvm/reference-vm/zephyr/Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/microtvm/reference-vm/zephyr/Vagrantfile b/apps/microtvm/reference-vm/zephyr/Vagrantfile index ca7d582ca57e..d76219a9a42d 100644 --- a/apps/microtvm/reference-vm/zephyr/Vagrantfile +++ b/apps/microtvm/reference-vm/zephyr/Vagrantfile @@ -16,7 +16,7 @@ # under the License. Vagrant.configure("2") do |config| - config.vm.box = "tlcpack/microtvm-zephyr-2.7-staging" + config.vm.box = "tlcpack/microtvm-zephyr-2.7" if ENV.has_key?("TVM_RVM_NUM_CORES") num_cores = ENV["TVM_RVM_NUM_CORES"]