From 06b84349d684d040f2f17ebfb57472dddf4bbe32 Mon Sep 17 00:00:00 2001 From: Julian Oes Date: Fri, 23 Sep 2022 17:41:16 +1200 Subject: [PATCH 1/6] mavlink: install pymavlink dependencies This automatically installs the pip packages lxml and future as required. --- third_party/mavlink/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/third_party/mavlink/CMakeLists.txt b/third_party/mavlink/CMakeLists.txt index b432a64dfa..0be03b6d2a 100644 --- a/third_party/mavlink/CMakeLists.txt +++ b/third_party/mavlink/CMakeLists.txt @@ -17,6 +17,8 @@ ExternalProject_add( GIT_TAG 0b5b5dabf9c6ae22f2f93f0db9a4e19dcf6f19ac PREFIX mavlink CONFIGURE_COMMAND Python3::Interpreter + -m pip install -r pymavlink/requirements.txt + COMMAND Python3::Interpreter -m pymavlink.tools.mavgen --lang=C --wire-protocol=2.0 From 4ae546314c9a107bf3b13c52fa5d3b77cd6a506f Mon Sep 17 00:00:00 2001 From: Julian Oes Date: Sat, 24 Sep 2022 12:03:35 +1200 Subject: [PATCH 2/6] workflows: install pip but not future We now install future and lxml using pip at configure time, so we should not need to install it beforehand. However, we do require pip. --- .github/workflows/main.yml | 35 ++++------------------------------- 1 file changed, 4 insertions(+), 31 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4f0935f5b7..448c7e53d8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,8 +31,6 @@ jobs: run: echo "superbuild=-DSUPERBUILD=OFF" >> $GITHUB_ENV && echo "cmake_prefix_path=-DCMAKE_PREFIX_PATH=$(pwd)/build/third_party/install" >> $GITHUB_ENV - name: Install lcov run: sudo apt-get update && sudo apt-get install -y lcov - - name: install pymavlink dependencies - run: sudo apt-get install -y python3-future - name: configure run: cmake $superbuild $cmake_prefix_path -DCMAKE_BUILD_TYPE=Coverage -DASAN=ON -DWERROR=OFF -Bbuild -H. - name: build @@ -61,8 +59,6 @@ jobs: submodules: recursive - name: install run: sudo apt-get update && sudo apt-get install -y libjsoncpp-dev libcurl4-openssl-dev libtinyxml2-dev - - name: install pymavlink dependencies - run: sudo apt-get install -y python3-future - name: configure run: cmake -DCMAKE_BUILD_TYPE=Release -DSUPERBUILD=OFF -DWERROR=OFF -Bbuild/release -H. - name: build @@ -90,8 +86,6 @@ jobs: - name: disable superbuild on cache hit if: steps.cache.outputs.cache-hit == 'true' run: echo "superbuild=-DSUPERBUILD=OFF" >> $GITHUB_ENV && echo "cmake_prefix_path=-DCMAKE_PREFIX_PATH=$(pwd)/build/release/third_party/install" >> $GITHUB_ENV - - name: install pymavlink dependencies - run: sudo apt-get update && sudo apt-get install -y python3-future - name: configure run: cmake $superbuild $cmake_prefix_path -DCMAKE_BUILD_TYPE=Release -DBUILD_MAVSDK_SERVER=ON -DWERROR=OFF -Bbuild/release -H. - name: build @@ -130,8 +124,6 @@ jobs: with: path: ~/.hunter key: ${{ github.job }}-${{ hashFiles('~/.hunter/**') }} - - name: install pymavlink dependencies - run: sudo apt-get update && sudo apt-get install -y python3-future - name: prepare mavlink headers because this is without superbuild run: tools/generate_mavlink_headers.sh - name: configure @@ -152,8 +144,6 @@ jobs: submodules: recursive - name: install clang-format run: sudo apt-get update && sudo apt-get install -y clang-format-12 - - name: install pymavlink dependencies - run: sudo apt-get install -y python3-future - uses: actions/cache@v2 id: cache with: @@ -180,8 +170,6 @@ jobs: submodules: recursive - name: install dependencies run: sudo apt-get update && sudo apt-get install -y doxygen clang-format-12 - - name: install pymavlink dependencies - run: sudo apt-get install -y python3-future - name: check style run: ./tools/fix_style.sh . - name: check docs @@ -209,9 +197,7 @@ jobs: wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | sudo apt-key add - sudo apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' sudo apt-get update - sudo apt-get install -y cmake - - name: install pymavlink dependencies - run: sudo apt-get install -y python3-future + sudo apt-get install -y cmake python3-pip - name: configure run: cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_MAVSDK_SERVER=OFF -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=install -DWERROR=OFF -Bbuild/release -H. - name: build @@ -239,8 +225,6 @@ jobs: fetch-depth: 0 - name: install packages run: sudo apt-get update && sudo apt-get install -y build-essential debhelper fakeroot libjsoncpp-dev libcurl4-openssl-dev libtinyxml2-dev libjsoncpp1 libcurl4 libtinyxml2-6a - - name: install pymavlink dependencies - run: sudo apt-get install -y python3-future - name: prepare mavlink headers because this is without superbuild run: tools/generate_mavlink_headers.sh - name: generate changelog @@ -292,8 +276,6 @@ jobs: - uses: actions/checkout@v2 with: submodules: recursive - - name: install pymavlink dependencies - run: sudo apt-get update && sudo apt-get install -y python3 python3-future - name: prepare mavlink headers run: tools/generate_mavlink_headers.sh - name: setup dockcross @@ -328,7 +310,7 @@ jobs: container: alpine:3.16.0 steps: - name: install tools - run: apk update && apk add build-base cmake git linux-headers perl tar python3 py3-future + run: apk update && apk add build-base cmake git linux-headers perl tar python3 py3-pip - uses: actions/checkout@v2 with: submodules: recursive @@ -381,7 +363,7 @@ jobs: with: submodules: recursive - name: install pymavlink dependencies - run: sudo apt-get update && sudo apt-get install -y python3 python3-future + run: sudo apt-get update && sudo apt-get install -y python3 python3-pip - name: prepare mavlink headers run: tools/generate_mavlink_headers.sh - name: setup dockcross @@ -427,7 +409,7 @@ jobs: with: submodules: recursive - name: install pymavlink dependencies - run: sudo apt-get update && sudo apt-get install -y python3 python3-future + run: sudo apt-get update && sudo apt-get install -y python3 python3-pip - name: prepare mavlink headers run: tools/generate_mavlink_headers.sh - name: setup dockcross @@ -482,8 +464,6 @@ jobs: run: echo "superbuild=-DSUPERBUILD=OFF" >> $GITHUB_ENV && echo "cmake_prefix_path=-DCMAKE_PREFIX_PATH=$(pwd)/build/macos/third_party/install" >> $GITHUB_ENV - name: set SDKROOT value run: echo "SDKROOT=$(xcrun --sdk macosx --show-sdk-path)" >> $GITHUB_ENV - - name: install pymavlink dependencies - run: python3 -m pip install future - name: configure run: cmake $superbuild $cmake_prefix_path -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=build/macos/install -DBUILD_MAVSDK_SERVER=ON -DBUILD_SHARED_LIBS=OFF -DMACOS_FRAMEWORK=${{ matrix.build-framework }} -DWERROR=OFF -Bbuild/macos -H. - name: build @@ -540,8 +520,6 @@ jobs: - name: set SDK-related environment variables (for non-cmake deps) run: | echo "SDKROOT=$(xcrun --sdk ${{ matrix.sdk }} --show-sdk-path)" >> $GITHUB_ENV - - name: install pymavlink dependencies - run: python3 -m pip install future - name: configure run: cmake $superbuild $cmake_prefix_path -DENABLE_STRICT_TRY_COMPILE=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=$(pwd)/tools/ios.toolchain.cmake -DPLATFORM=${{ matrix.platform }} -DDEPLOYMENT_TARGET=13.0 -DBUILD_MAVSDK_SERVER=ON -DBUILD_SHARED_LIBS=OFF -DWERROR=OFF -Bbuild/${{ matrix.name }} -H. - name: build @@ -623,7 +601,6 @@ jobs: run: | python3 -m pip install --upgrade pip python3 -m pip install wheel - python3 -m pip install future - name: configure run: cmake -G "Visual Studio 17 2022" $env:superbuild $env:cmake_prefix_path -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=build/release/install -DBUILD_MAVSDK_SERVER=ON -DBUILD_SHARED_LIBS=OFF -DWERROR=OFF -Bbuild/release -S. - name: build @@ -666,8 +643,6 @@ jobs: run: | git config --global --add safe.directory "$GITHUB_WORKSPACE" git config --global --add safe.directory /github/workspace - - name: install pymavlink dependencies - run: python3 -m pip install future - name: test run: PX4_VERSION=${{ matrix.px4_version }} tools/run-sitl-tests.sh /home/user/Firmware timeout-minutes: 45 @@ -700,8 +675,6 @@ jobs: run: | git config --global --add safe.directory "$GITHUB_WORKSPACE" git config --global --add safe.directory /github/workspace - - name: install pymavlink dependencies - run: python3 -m pip install future - name: test run: APM_VERSION=${{ matrix.apm_version }} tools/run-sitl-tests.sh /home/user/ArduPilot timeout-minutes: 45 From e95099e4181680eaab1820f4267dbd04a4716d43 Mon Sep 17 00:00:00 2001 From: Julian Oes Date: Sat, 24 Sep 2022 12:31:57 +1200 Subject: [PATCH 3/6] tools: install dependencies for pymavlink --- tools/generate_mavlink_headers.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/generate_mavlink_headers.sh b/tools/generate_mavlink_headers.sh index 6b0ceadc96..f97c2126cc 100755 --- a/tools/generate_mavlink_headers.sh +++ b/tools/generate_mavlink_headers.sh @@ -23,6 +23,12 @@ OUTPUT_PATH="$MAVLINK_HEADERS_PATH/include/mavlink/v2.0/" mkdir -p "$OUTPUT_PATH" OUTPUT_PATH=$(realpath $OUTPUT_PATH) +echo "Installing dependencies" +(cd $MAVLINK_PATH && \ + python3 \ + -m pip install -r pymavlink/requirements.txt) + +echo "Generating headers" (cd $MAVLINK_PATH && \ python3 \ -m pymavlink.tools.mavgen \ From 93fbaa483ceeb9208b9c696cc6d236e04ad16f6d Mon Sep 17 00:00:00 2001 From: Julian Oes Date: Tue, 8 Nov 2022 14:58:41 +1300 Subject: [PATCH 4/6] docker: Fedora 34 is EOL --- docker/Dockerfile-Fedora-34 | 56 -------------------------- docker/Dockerfile-Fedora-35 | 2 +- docker/Dockerfile-Fedora-36 | 2 +- docker/build_and_push_docker_images.sh | 2 - 4 files changed, 2 insertions(+), 60 deletions(-) delete mode 100644 docker/Dockerfile-Fedora-34 diff --git a/docker/Dockerfile-Fedora-34 b/docker/Dockerfile-Fedora-34 deleted file mode 100644 index 698ecf5c25..0000000000 --- a/docker/Dockerfile-Fedora-34 +++ /dev/null @@ -1,56 +0,0 @@ -# -# Development environment for the MAVSDK based on Fedora 34. -# -# Author: Julian Oes -# - -FROM fedora:34 - -MAINTAINER Julian Oes - -RUN dnf -y install \ - autoconf \ - automake \ - ccache \ - clang \ - cmake \ - colordiff \ - doxygen \ - gcc \ - gcc-c++ \ - git \ - golang \ - libcurl-devel \ - libtool \ - make \ - ninja-build \ - perl-FindBin \ - python \ - python-future \ - redhat-rpm-config \ - rpm-build \ - ruby-devel \ - rubygems \ - sudo \ - tinyxml2-devel \ - wget \ - which \ - zlib-devel \ - && dnf clean all - -RUN gem install --no-document fpm; - -RUN wget -qO- https://github.com/ncopa/su-exec/archive/dddd1567b7c76365e1e0aac561287975020a8fad.tar.gz | tar xvz && \ - cd su-exec-* && make && mv su-exec /usr/local/bin && cd .. && rm -rf su-exec-* - -# Create user with id 1001 (Jenkins docker workflow default) -RUN useradd --shell /bin/bash -u 1001 -c "" -m user - -ADD /sudoers.txt /etc/sudoers -RUN chmod 440 /etc/sudoers - -COPY entrypoint.sh /usr/local/bin/entrypoint.sh - -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] - -WORKDIR "/home/user/MAVSDK" diff --git a/docker/Dockerfile-Fedora-35 b/docker/Dockerfile-Fedora-35 index 2516be1f03..c1560d0733 100644 --- a/docker/Dockerfile-Fedora-35 +++ b/docker/Dockerfile-Fedora-35 @@ -26,7 +26,7 @@ RUN dnf -y install \ ninja-build \ perl-FindBin \ python \ - python-future \ + python-pip \ redhat-rpm-config \ rpm-build \ ruby-devel \ diff --git a/docker/Dockerfile-Fedora-36 b/docker/Dockerfile-Fedora-36 index 88a6a99a42..de92c9cda0 100644 --- a/docker/Dockerfile-Fedora-36 +++ b/docker/Dockerfile-Fedora-36 @@ -26,7 +26,7 @@ RUN dnf -y install \ ninja-build \ perl-FindBin \ python \ - python-future \ + python-pip \ redhat-rpm-config \ rpm-build \ ruby-devel \ diff --git a/docker/build_and_push_docker_images.sh b/docker/build_and_push_docker_images.sh index ea4f5b587e..93f38b62e4 100755 --- a/docker/build_and_push_docker_images.sh +++ b/docker/build_and_push_docker_images.sh @@ -10,7 +10,6 @@ then DOCKER_CMD=podman fi -$DOCKER_CMD build -f Dockerfile-Fedora-34 -t mavsdk/mavsdk-fedora-34 . $DOCKER_CMD build -f Dockerfile-Fedora-35 -t mavsdk/mavsdk-fedora-35 . $DOCKER_CMD build -f Dockerfile-Fedora-36 -t mavsdk/mavsdk-fedora-36 . $DOCKER_CMD build -f Dockerfile-Ubuntu-20.04 -t mavsdk/mavsdk-ubuntu-20.04 . @@ -24,7 +23,6 @@ $DOCKER_CMD build -f Dockerfile.dockcross-linux-armv6-custom -t mavsdk/mavsdk-do $DOCKER_CMD build -f Dockerfile.dockcross-linux-armv7-custom -t mavsdk/mavsdk-dockcross-linux-armv7-custom . $DOCKER_CMD build -f Dockerfile.dockcross-linux-arm64-custom -t mavsdk/mavsdk-dockcross-linux-arm64-custom . -$DOCKER_CMD push mavsdk/mavsdk-fedora-34:latest $DOCKER_CMD push mavsdk/mavsdk-fedora-35:latest $DOCKER_CMD push mavsdk/mavsdk-fedora-36:latest $DOCKER_CMD push mavsdk/mavsdk-ubuntu-20.04:latest From 3c2d07021e1fb098b12f406edb998a54bce36b03 Mon Sep 17 00:00:00 2001 From: Julian Oes Date: Thu, 10 Nov 2022 13:30:09 +1300 Subject: [PATCH 5/6] workflows: an attempt to fix pip builds on iOS --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 448c7e53d8..3bd6c02415 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -520,6 +520,10 @@ jobs: - name: set SDK-related environment variables (for non-cmake deps) run: | echo "SDKROOT=$(xcrun --sdk ${{ matrix.sdk }} --show-sdk-path)" >> $GITHUB_ENV + - name: install pymavlink dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install wheel - name: configure run: cmake $superbuild $cmake_prefix_path -DENABLE_STRICT_TRY_COMPILE=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=$(pwd)/tools/ios.toolchain.cmake -DPLATFORM=${{ matrix.platform }} -DDEPLOYMENT_TARGET=13.0 -DBUILD_MAVSDK_SERVER=ON -DBUILD_SHARED_LIBS=OFF -DWERROR=OFF -Bbuild/${{ matrix.name }} -H. - name: build From 3b7e13eb351e927ffc39d3ec419e7290b259b767 Mon Sep 17 00:00:00 2001 From: Julian Oes Date: Fri, 11 Nov 2022 14:42:51 +1300 Subject: [PATCH 6/6] workflows: another attempt at buliding for iOS --- .github/workflows/main.yml | 1 + third_party/mavlink/CMakeLists.txt | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3bd6c02415..83356b9ede 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -524,6 +524,7 @@ jobs: run: | python3 -m pip install --upgrade pip python3 -m pip install wheel + python3 -m pip install future - name: configure run: cmake $superbuild $cmake_prefix_path -DENABLE_STRICT_TRY_COMPILE=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=$(pwd)/tools/ios.toolchain.cmake -DPLATFORM=${{ matrix.platform }} -DDEPLOYMENT_TARGET=13.0 -DBUILD_MAVSDK_SERVER=ON -DBUILD_SHARED_LIBS=OFF -DWERROR=OFF -Bbuild/${{ matrix.name }} -H. - name: build diff --git a/third_party/mavlink/CMakeLists.txt b/third_party/mavlink/CMakeLists.txt index 0be03b6d2a..5a69ee989d 100644 --- a/third_party/mavlink/CMakeLists.txt +++ b/third_party/mavlink/CMakeLists.txt @@ -11,13 +11,18 @@ if (NOT MAVLINK_DIALECT) set(MAVLINK_DIALECT common) endif() + +if(NOT CMAKE_CROSSCOMPILING) +set(CONFIGURE_COMMAND + Python3::Interpreter -m pip install -r pymavlink/requirements.txt) +endif() + ExternalProject_add( mavlink GIT_REPOSITORY https://github.com/mavlink/mavlink GIT_TAG 0b5b5dabf9c6ae22f2f93f0db9a4e19dcf6f19ac PREFIX mavlink - CONFIGURE_COMMAND Python3::Interpreter - -m pip install -r pymavlink/requirements.txt + CONFIGURE_COMMAND ${CONFIGURE_COMMAND} COMMAND Python3::Interpreter -m pymavlink.tools.mavgen --lang=C