From 12259932789402a65cc8c038e2fdee3ce00ef22c Mon Sep 17 00:00:00 2001 From: Alessandro Gario Date: Thu, 18 Feb 2021 16:50:15 +0100 Subject: [PATCH 1/3] CMake: Add packaging support --- .github/workflows/vcpkg_ci.yml | 6 ++- packaging/README.md | 17 +++++++++ packaging/cmake/dispatcher.cmake | 23 ++++++++++++ .../cmake/system/linux/generators/deb.cmake | 12 ++++++ .../cmake/system/linux/generators/rpm.cmake | 13 +++++++ .../cmake/system/linux/generators/tgz.cmake | 10 +++++ .../cmake/system/macos/generators/tgz.cmake | 10 +++++ packaging/main.cmake | 30 +++++++++++++++ scripts/build_with_vcpkg.sh | 37 ++++++++++++++++++- 9 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 packaging/README.md create mode 100644 packaging/cmake/dispatcher.cmake create mode 100644 packaging/cmake/system/linux/generators/deb.cmake create mode 100644 packaging/cmake/system/linux/generators/rpm.cmake create mode 100644 packaging/cmake/system/linux/generators/tgz.cmake create mode 100644 packaging/cmake/system/macos/generators/tgz.cmake create mode 100644 packaging/main.cmake diff --git a/.github/workflows/vcpkg_ci.yml b/.github/workflows/vcpkg_ci.yml index 3f02e8bf..29d7eb2d 100644 --- a/.github/workflows/vcpkg_ci.yml +++ b/.github/workflows/vcpkg_ci.yml @@ -31,12 +31,14 @@ jobs: steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Install utility tools shell: bash run: | # TODO some of these should probably live in the Docker build image apt-get update - apt-get install -y pixz xz-utils make + apt-get install -y pixz xz-utils make rpm - name: Build with build script shell: bash @@ -69,6 +71,8 @@ jobs: steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Build with build script shell: bash run: | diff --git a/packaging/README.md b/packaging/README.md new file mode 100644 index 00000000..938edb8a --- /dev/null +++ b/packaging/README.md @@ -0,0 +1,17 @@ +# rellic packaging scripts + +## How to generate packages + +1. Configure and build rellic +2. Set the **DESTDIR** variable to a new folder +3. Run the packaging script, passing the **DESTDIR** folder + +Example: + +```sh +rellic_version=$(git describe --always) + +cpack -D RELLIC_DATA_PATH="/path/to/install/directory" \ + -R ${rellic_version} \ + --config "packaging/main.cmake" +``` diff --git a/packaging/cmake/dispatcher.cmake b/packaging/cmake/dispatcher.cmake new file mode 100644 index 00000000..0df79805 --- /dev/null +++ b/packaging/cmake/dispatcher.cmake @@ -0,0 +1,23 @@ +# +# Copyright (c) 2021-present, Trail of Bits, Inc. +# All rights reserved. +# +# This source code is licensed in accordance with the terms specified in +# the LICENSE file found in the root directory of this source tree. +# + +set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${CMAKE_SYSTEM_PROCESSOR}") +set(CPACK_INSTALLED_DIRECTORIES "${RELLIC_DATA_PATH};.") + +string(TOLOWER "${CMAKE_SYSTEM_NAME}" system_name) +if(system_name STREQUAL "darwin") + set(system_name "macos") +endif() + +set(common_include "${CMAKE_CURRENT_LIST_DIR}/system/${system_name}/common.cmake") +if(EXISTS "${common_include}") + include("${common_include}") +endif() + +string(TOLOWER "${CPACK_GENERATOR}" cpack_generator) +include("${CMAKE_CURRENT_LIST_DIR}/system/${system_name}/generators/${cpack_generator}.cmake") diff --git a/packaging/cmake/system/linux/generators/deb.cmake b/packaging/cmake/system/linux/generators/deb.cmake new file mode 100644 index 00000000..17796320 --- /dev/null +++ b/packaging/cmake/system/linux/generators/deb.cmake @@ -0,0 +1,12 @@ +# +# Copyright (c) 2021-present, Trail of Bits, Inc. +# All rights reserved. +# +# This source code is licensed in accordance with the terms specified in +# the LICENSE file found in the root directory of this source tree. +# + +set(CPACK_DEBIAN_PACKAGE_PRIORITY "extra") +set(CPACK_DEBIAN_PACKAGE_SECTION "default") +set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "${CPACK_PACKAGE_HOMEPAGE_URL}") + diff --git a/packaging/cmake/system/linux/generators/rpm.cmake b/packaging/cmake/system/linux/generators/rpm.cmake new file mode 100644 index 00000000..16331a11 --- /dev/null +++ b/packaging/cmake/system/linux/generators/rpm.cmake @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021-present, Trail of Bits, Inc. +# All rights reserved. +# +# This source code is licensed in accordance with the terms specified in +# the LICENSE file found in the root directory of this source tree. +# + +set(CPACK_RPM_PACKAGE_RELEASE "${CPACK_PACKAGE_VERSION}") +set(CPACK_RPM_FILE_NAME "RPM-DEFAULT") +set(CPACK_RPM_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION}") +set(CPACK_RPM_PACKAGE_GROUP "default") +set(CPACK_RPM_PACKAGE_LICENSE "GNU Affero General Public License v3.0") diff --git a/packaging/cmake/system/linux/generators/tgz.cmake b/packaging/cmake/system/linux/generators/tgz.cmake new file mode 100644 index 00000000..f3356eb4 --- /dev/null +++ b/packaging/cmake/system/linux/generators/tgz.cmake @@ -0,0 +1,10 @@ +# +# Copyright (c) 2021-present, Trail of Bits, Inc. +# All rights reserved. +# +# This source code is licensed in accordance with the terms specified in +# the LICENSE file found in the root directory of this source tree. +# + +set(CPACK_SET_DESTDIR ON) +set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY 0) diff --git a/packaging/cmake/system/macos/generators/tgz.cmake b/packaging/cmake/system/macos/generators/tgz.cmake new file mode 100644 index 00000000..f3356eb4 --- /dev/null +++ b/packaging/cmake/system/macos/generators/tgz.cmake @@ -0,0 +1,10 @@ +# +# Copyright (c) 2021-present, Trail of Bits, Inc. +# All rights reserved. +# +# This source code is licensed in accordance with the terms specified in +# the LICENSE file found in the root directory of this source tree. +# + +set(CPACK_SET_DESTDIR ON) +set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY 0) diff --git a/packaging/main.cmake b/packaging/main.cmake new file mode 100644 index 00000000..4e6bb374 --- /dev/null +++ b/packaging/main.cmake @@ -0,0 +1,30 @@ +# +# Copyright (c) 2021-present, Trail of Bits, Inc. +# All rights reserved. +# +# This source code is licensed in accordance with the terms specified in +# the LICENSE file found in the root directory of this source tree. +# + +if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") + set(CPACK_GENERATOR "TGZ;DEB;RPM") + +elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") + set(CPACK_GENERATOR "TGZ") +endif() + +if(RELLIC_DATA_PATH STREQUAL "") + message(FATAL_ERROR "The RELLIC_DATA_PATH variable was not set") +endif() + +if(RELLIC_PACKAGE_VERSION STREQUAL "") + message(FATAL_ERROR "The RELLIC_PACKAGE_VERSION variable was not set") +endif() + +set(CPACK_PROJECT_CONFIG_FILE "${CMAKE_CURRENT_LIST_DIR}/cmake/dispatcher.cmake") + +set(CPACK_PACKAGE_DESCRIPTION "Rellic produces goto-free C output from LLVM bitcode") +set(CPACK_PACKAGE_NAME "rellic") +set(CPACK_PACKAGE_VENDOR "Trail of Bits") +set(CPACK_PACKAGE_CONTACT "opensource@trailofbits.com") +set(CPACK_PACKAGE_HOMEPAGE_URL "https://github.com/lifting-bits/rellic") diff --git a/scripts/build_with_vcpkg.sh b/scripts/build_with_vcpkg.sh index 1320e17d..65d6b23f 100755 --- a/scripts/build_with_vcpkg.sh +++ b/scripts/build_with_vcpkg.sh @@ -264,6 +264,41 @@ function Build return $? } +# Create the packages +function Package +{ + tag_count=$(cd "${SRC_DIR}" && git tag | wc -l) + if [[ ${tag_count} == 0 ]]; then + echo "WARNING: No tag found, marking this release as 0.0.0" + rellic_tag="v0.0.0" + else + rellic_tag=$(cd "${SRC_DIR}" && git describe --tags --always --abbrev=0) + fi + + rellic_commit=$(cd "${SRC_DIR}" && git rev-parse HEAD | cut -c1-7) + rellic_version="${rellic_tag:1}.${rellic_commit}" + + ( + set -x + + if [[ -d "install" ]]; then + rm -rf "install" + fi + + mkdir "install" + export DESTDIR="$(pwd)/install" + + cmake --build . \ + --target install + + cpack -D RELLIC_DATA_PATH="${DESTDIR}" \ + -R ${rellic_version} \ + --config "${SRC_DIR}/packaging/main.cmake" + ) || return $? + + return $? +} + # Get a LLVM version name for the build. This is used to find the version of # cxx-common to download. function GetLLVMVersion @@ -375,7 +410,7 @@ function main mkdir -p "${BUILD_DIR}" cd "${BUILD_DIR}" || exit 1 - if ! (DownloadLibraries && Configure && Build); then + if ! (DownloadLibraries && Configure && Build && Package); then echo "[x] Build aborted." exit 1 fi From c121a09fa34cb7f74de92c0d6d58fe8ce03987fe Mon Sep 17 00:00:00 2001 From: Alessandro Gario Date: Thu, 18 Feb 2021 19:44:08 +0100 Subject: [PATCH 2/3] CI: Upload artifacts, auto-create releases from tags --- .github/workflows/vcpkg_ci.yml | 138 +++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/.github/workflows/vcpkg_ci.yml b/.github/workflows/vcpkg_ci.yml index 29d7eb2d..f8d5a48f 100644 --- a/.github/workflows/vcpkg_ci.yml +++ b/.github/workflows/vcpkg_ci.yml @@ -56,6 +56,33 @@ jobs: CLANG_BIN="$(find ../../pre-built-llvm-${{ matrix.llvm }} -name clang | grep "bin/clang")" ../scripts/roundtrip.py "./tools/rellic-decomp-${{ matrix.llvm }}.0" ../tests/tools/decomp "${CLANG_BIN}" + - name: Locate the packages + id: package_names + shell: bash + working-directory: rellic-build + run: | + echo ::set-output name=DEB_PACKAGE_PATH::rellic-build/$(ls *.deb) + echo ::set-output name=RPM_PACKAGE_PATH::rellic-build/$(ls *.rpm) + echo ::set-output name=TGZ_PACKAGE_PATH::rellic-build/$(ls *.tar.gz) + + - name: Store the DEB package + uses: actions/upload-artifact@v1 + with: + name: ${{ matrix.image.name }}-${{ matrix.image.tag }}_llvm${{ matrix.llvm }}_deb_package + path: ${{ steps.package_names.outputs.DEB_PACKAGE_PATH }} + + - name: Store the RPM package + uses: actions/upload-artifact@v1 + with: + name: ${{ matrix.image.name }}-${{ matrix.image.tag }}_llvm${{ matrix.llvm }}_rpm_package + path: ${{ steps.package_names.outputs.RPM_PACKAGE_PATH }} + + - name: Store the TGZ package + uses: actions/upload-artifact@v1 + with: + name: ${{ matrix.image.name }}-${{ matrix.image.tag }}_llvm${{ matrix.llvm }}_tgz_package + path: ${{ steps.package_names.outputs.TGZ_PACKAGE_PATH }} + build_mac: strategy: fail-fast: false @@ -88,3 +115,114 @@ jobs: # Run tests manually (kinda ugly, so use CMake way) CLANG_BIN="$(find ../../pre-built-llvm-${{ matrix.llvm }} -name clang | grep "bin/clang")" ../scripts/roundtrip.py "./tools/rellic-decomp-${{ matrix.llvm }}.0" ../tests/tools/decomp "${CLANG_BIN}" + - name: Locate the packages + id: package_names + shell: bash + working-directory: rellic-build + run: | + echo ::set-output name=TGZ_PACKAGE_PATH::rellic-build/$(ls *.tar.gz) + + - name: Store the TGZ package + uses: actions/upload-artifact@v1 + with: + name: macos-11.0_llvm${{ matrix.llvm }}_tgz_package + path: ${{ steps.package_names.outputs.TGZ_PACKAGE_PATH }} + + + release_linux: + # Do not run the release procedure if any of the builds has failed + needs: [ build_linux, build_mac ] + runs-on: ubuntu-20.04 + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + + steps: + - name: Download all artifacts + uses: actions/download-artifact@v2 + + - name: Draft the new release + id: create_release + uses: actions/create-release@v1 + + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + with: + tag_name: ${{ github.ref }} + release_name: Version ${{ github.ref }} + draft: true + prerelease: true + + - name: Group the packages by platform + run: | + zip -r9 rellic_ubuntu-18.04_packages.zip \ + ubuntu-18.04* + + zip -r9 rellic_ubuntu-20.04_packages.zip \ + ubuntu-20.04* + + - name: Upload the Ubuntu 18.04 packages + uses: actions/upload-release-asset@v1 + + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: rellic_ubuntu-18.04_packages.zip + asset_name: rellic_ubuntu-18.04_packages.zip + asset_content_type: application/gzip + + - name: Upload the Ubuntu 20.04 packages + uses: actions/upload-release-asset@v1 + + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: rellic_ubuntu-20.04_packages.zip + asset_name: rellic_ubuntu-20.04_packages.zip + asset_content_type: application/gzip + + + + + release_macos: + # Do not run the release procedure if any of the builds has failed + needs: [ build_linux, build_mac ] + runs-on: 'macos-11.0' + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + + steps: + - name: Download all artifacts + uses: actions/download-artifact@v2 + + - name: Draft the new release + id: create_release + uses: actions/create-release@v1 + + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + with: + tag_name: ${{ github.ref }} + release_name: Version ${{ github.ref }} + draft: true + prerelease: true + + - name: Group the packages by platform + run: | + zip -r9 rellic_macos-11.0_packages.zip \ + macos-11.0* + + - name: Upload the macOS 11.0 packages + uses: actions/upload-release-asset@v1 + + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: rellic_macos-11.0_packages.zip + asset_name: rellic_macos-11.0_packages.zip + asset_content_type: application/gzip From bcd634d0ee4be1da45ba61281f34e1e4d1bb8fa4 Mon Sep 17 00:00:00 2001 From: Alessandro Gario Date: Thu, 18 Feb 2021 21:00:45 +0100 Subject: [PATCH 3/3] Build: Lock the cxx-common version --- scripts/build.sh | 3 ++- scripts/build_with_vcpkg.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index 339e9d11..d5e80581 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -28,6 +28,7 @@ ARCH_VERSION=unknown BUILD_FLAGS= USE_HOST_COMPILER=0 LIBRARIES="${BUILD_DIR}/libraries" +CXX_COMMON_VERSION="v0.1.1" Z3_ARCHIVE=z3- Z3_VERSION=4.7.1 @@ -102,7 +103,7 @@ function GetArchVersion function DownloadCxxCommon { local GITHUB_LIBS="${LIBRARY_VERSION}.tar.xz" - local URL="https://github.com/trailofbits/cxx-common/releases/latest/download/${GITHUB_LIBS}" + local URL="https://github.com/trailofbits/cxx-common/releases/download/${CXX_COMMON_VERSION}/${GITHUB_LIBS}" echo "Fetching: ${URL}" if ! curl -LO "${URL}"; then diff --git a/scripts/build_with_vcpkg.sh b/scripts/build_with_vcpkg.sh index 65d6b23f..21c1af37 100755 --- a/scripts/build_with_vcpkg.sh +++ b/scripts/build_with_vcpkg.sh @@ -28,6 +28,7 @@ LLVM_VERSION=llvm-9 OS_VERSION=unknown ARCH_VERSION=unknown BUILD_FLAGS= +CXX_COMMON_VERSION="v0.1.1" # There are pre-build versions of various libraries for specific # Ubuntu releases. @@ -102,7 +103,7 @@ function GetArchVersion function DownloadVcpkgLibraries { local GITHUB_LIBS="${LIBRARY_VERSION}.tar.xz" - local URL="https://github.com/trailofbits/cxx-common/releases/latest/download/${GITHUB_LIBS}" + local URL="https://github.com/trailofbits/cxx-common/releases/download/${CXX_COMMON_VERSION}/${GITHUB_LIBS}" mkdir -p "${DOWNLOAD_DIR}" pushd "${DOWNLOAD_DIR}" || return 1