diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 7a63a2270..d522ba487 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -5,10 +5,11 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} on: - pull_request: push: branches: ["main"] tags: ["**"] + pull_request: + branches: ["main"] jobs: macos: @@ -16,38 +17,40 @@ jobs: runs-on: macos-latest strategy: matrix: - LLVM_VERSION: [13, 14, 15, 16, 17, 18] - + include: + - OS_NAME: "macos" + LLVM_VERSION: 13 + - OS_NAME: "macos" + LLVM_VERSION: 14 + - OS_NAME: "macos" + LLVM_VERSION: 15 + - OS_NAME: "macos" + LLVM_VERSION: 16 + - OS_NAME: "macos" + LLVM_VERSION: 17 + - OS_NAME: "macos" + LLVM_VERSION: 18 steps: - - name: Debugging - run: | - env | sort - cat $GITHUB_EVENT_PATH - - name: Fix git run: | git config --global --add safe.directory $PWD - - - name: Checkout sources - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - - uses: ./actions/detect-package-metadata id: metadata - name: Run CI task run: | - brew install ansible - cd infrastructure && \ - ansible-playbook macos-playbook.yaml \ - -e llvm_version="${{ matrix.LLVM_VERSION }}" \ - -e source_dir=$PWD/.. \ - -e gitref=$GITHUB_SHA \ - -e host=localhost \ - -e SDKROOT=`xcrun -show-sdk-path` \ - -e mull_version=${{ steps.metadata.outputs.version }}${{ steps.metadata.outputs.suffix }} \ - --verbose + brew install zlib ncurses sqlite ninja llvm@${{ matrix.LLVM_VERSION }} + pip3 install -r requirements.txt --break-system-packages + ./infrastructure/generator.py cmake --os macos --llvm_version ${{ matrix.LLVM_VERSION }} + cmake --preset mull -DMULL_VERSION=${{ steps.metadata.outputs.version }}${{ steps.metadata.outputs.suffix }} + cmake --workflow --preset build-and-test + cmake --workflow --preset package + ./end2end-tests/setup_end2end_tests.sh + ./end2end-tests/run_end2end_tests.sh macos ${{ matrix.LLVM_VERSION }} + mull-runner-${{ matrix.LLVM_VERSION }} --version - name: Publish package if: env.CLOUDSMITH_API_KEY != null @@ -57,12 +60,12 @@ jobs: --version ${{ steps.metadata.outputs.version }}${{ steps.metadata.outputs.suffix }} \ mull-project/mull-${{ steps.metadata.outputs.channel }} \ --tags macos \ - infrastructure/packages/`cat infrastructure/PACKAGE_FILE_NAME`.zip + build.${{ matrix.OS_NAME }}.${{ matrix.LLVM_VERSION }}.dir/*.zip - name: Move package run: | mkdir -p /tmp/packages - mv infrastructure/packages/`cat infrastructure/PACKAGE_FILE_NAME`.zip /tmp/packages/`cat infrastructure/PACKAGE_FILE_NAME`.zip + mv build.${{ matrix.OS_NAME }}.${{ matrix.LLVM_VERSION }}.dir/*.zip /tmp/packages/ - uses: ./actions/attach-package with: diff --git a/.github/workflows/ci-ubuntu.yml b/.github/workflows/ci-ubuntu.yml index 5f41978e7..33fdf665f 100644 --- a/.github/workflows/ci-ubuntu.yml +++ b/.github/workflows/ci-ubuntu.yml @@ -63,7 +63,7 @@ jobs: cmake --preset mull -DMULL_VERSION=${{ steps.metadata.outputs.version }}${{ steps.metadata.outputs.suffix }} cmake --workflow --preset build-and-test cmake --workflow --preset package - ./end2end-tests/run_end2end_tests.sh ${{ matrix.LLVM_VERSION }} + ./end2end-tests/run_end2end_tests.sh ${{ matrix.OS_NAME }} ${{ matrix.LLVM_VERSION }} mull-runner-${{ matrix.LLVM_VERSION }} --version - name: Publish package @@ -77,12 +77,12 @@ jobs: UBUNTU_CODENAME=$(. /etc/os-release; echo "${UBUNTU_CODENAME/*, /}") env CLOUDSMITH_API_KEY=${{ secrets.CLOUDSMITH_API_KEY }} cloudsmith push deb \ mull-project/mull-${{ steps.metadata.outputs.channel }}/${{ matrix.OS_NAME }}/${UBUNTU_CODENAME} \ - build.${{ matrix.LLVM_VERSION }}.dir/*.deb + build.${{ matrix.OS_NAME }}.${{ matrix.LLVM_VERSION }}.dir/*.deb - name: Move package run: | mkdir -p /tmp/packages - sudo mv build.${{ matrix.LLVM_VERSION }}.dir/*.deb /tmp/packages/ + sudo mv build.${{ matrix.OS_NAME }}.${{ matrix.LLVM_VERSION }}.dir/*.deb /tmp/packages/ - uses: ./actions/attach-package with: diff --git a/end2end-tests/run_end2end_tests.sh b/end2end-tests/run_end2end_tests.sh index cc376e514..306ca7d58 100755 --- a/end2end-tests/run_end2end_tests.sh +++ b/end2end-tests/run_end2end_tests.sh @@ -1,16 +1,30 @@ -#!/usr/bin/bash +#!/bin/bash set -e set -x -LLVM_VERSION=$1 +OS_NAME=$1 +LLVM_VERSION=$2 +BUILD_DIR=build.$OS_NAME.$LLVM_VERSION.dir if (( $LLVM_VERSION < 16 )); then EXTRA_FLAGS=-fexperimental-new-pass-manager fi -# Install mull system-wide -dpkg -i build.${LLVM_VERSION}.dir/*.deb +if [ "$OS_NAME" == "macos" ] +then + sudo cmake --install $BUILD_DIR + PLUGIN_PATH=/usr/local/lib/mull-ir-frontend-$LLVM_VERSION + MULL_CC=/opt/homebrew/opt/llvm@$LLVM_VERSION/bin/clang + MULL_CXX=/opt/homebrew/opt/llvm@$LLVM_VERSION/bin/clang++ +else + dpkg -i $BUILD_DIR/*.deb + PLUGIN_PATH=/usr/lib/mull-ir-frontend-$LLVM_VERSION + MULL_CC=clang-$LLVM_VERSION + MULL_CXX=clang++-$LLVM_VERSION +fi + +which mull-runner-$LLVM_VERSION pushd `dirname "$0"` @@ -23,10 +37,10 @@ function test_fmt() { pushd fmt.build.dir cmake -G Ninja \ - -DCMAKE_CXX_FLAGS="-grecord-command-line -fpass-plugin=/usr/lib/mull-ir-frontend-$LLVM_VERSION $EXTRA_FLAGS" \ + -DCMAKE_CXX_FLAGS="-grecord-command-line -fpass-plugin=$PLUGIN_PATH $EXTRA_FLAGS" \ -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_C_COMPILER=clang-$LLVM_VERSION \ - -DCMAKE_CXX_COMPILER=clang++-$LLVM_VERSION \ + -DCMAKE_C_COMPILER=$MULL_CC \ + -DCMAKE_CXX_COMPILER=$MULL_CXX \ .. ninja core-test @@ -42,13 +56,19 @@ function test_fmt() { function test_openssl() { pushd openssl - env CC=clang-$LLVM_VERSION ./config -g -O0 \ + env CC=$MULL_CC ./config -g -O0 \ $EXTRA_FLAGS \ -grecord-command-line \ - -fpass-plugin=/usr/lib/mull-ir-frontend-$LLVM_VERSION + -fpass-plugin=$PLUGIN_PATH make build_generated -j make ./test/bio_enc_test -j + + if [ "$OS_NAME" == "macos" ] + then + install_name_tool -change /usr/local/lib/libcrypto.3.dylib $PWD/libcrypto.3.dylib ./test/bio_enc_test + fi + env LD_LIBRARY_PATH=. ./test/bio_enc_test env LD_LIBRARY_PATH=. mull-runner-$LLVM_VERSION --allow-surviving ./test/bio_enc_test diff --git a/end2end-tests/setup_end2end_tests.sh b/end2end-tests/setup_end2end_tests.sh index f5df0204b..9c176164d 100755 --- a/end2end-tests/setup_end2end_tests.sh +++ b/end2end-tests/setup_end2end_tests.sh @@ -1,4 +1,4 @@ -#!/usr/bin/bash +#!/bin/bash set -e set -x diff --git a/infrastructure/generator.py b/infrastructure/generator.py index 003692832..e8613e048 100755 --- a/infrastructure/generator.py +++ b/infrastructure/generator.py @@ -6,38 +6,56 @@ import shutil SUPPORTED_PLATFORMS = { - "ubuntu": { - "20.04": [12], - "22.04": [13, 14, 15], - "24.04": [14, 15, 16, 17, 18] - } + "ubuntu": {"20.04": [12], "22.04": [13, 14, 15], "24.04": [14, 15, 16, 17, 18]}, + "macos": {"latest": [13, 14, 15, 16, 17, 18]}, } def cmake(args): - template_name = f"infrastructure/templates/cmake-presets/{args.os}.mustache" - template_args = {"LLVM_VERSION": args.llvm_version} + os_specific_args = { + "ubuntu": { + "CC": f"clang-{args.llvm_version}", + "CXX": f"clang++-{args.llvm_version}", + "CMAKE_PREFIX_PATH": f"/usr/lib/llvm-{args.llvm_version}/cmake/;/usr/lib/cmake/clang-{args.llvm_version}/", + }, + "macos": { + "CC": f"/opt/homebrew/opt/llvm@{args.llvm_version}/bin/clang", + "CXX": f"/opt/homebrew/opt/llvm@{args.llvm_version}/bin/clang++", + "CMAKE_PREFIX_PATH": f"/opt/homebrew/opt/llvm@{args.llvm_version}/lib/cmake/llvm/;/opt/homebrew/opt/llvm@{args.llvm_version}/lib/cmake/clang/", + }, + } + template_name = f"infrastructure/templates/cmake-presets/CMakePresets.json.mustache" + template_args = os_specific_args[args.os] + template_args["LLVM_VERSION"] = args.llvm_version + template_args["OS_NAME"] = args.os renderer = pystache.Renderer(missing_tags="strict") with open(template_name, "r") as t: result = renderer.render(t.read(), template_args) - with open('CMakePresets.json', "w") as f: + with open("CMakePresets.json", "w") as f: f.write(result) def devcontainers(args): shutil.rmtree(".devcontainer") - for (os_name, platform) in SUPPORTED_PLATFORMS.items(): + for os_name, platform in SUPPORTED_PLATFORMS.items(): + if os_name.lower() == "macos": + continue template_folder = f"infrastructure/templates/devcontainers/{os_name}" - for (os_version, llvm_versions) in platform.items(): + for os_version, llvm_versions in platform.items(): for llvm_version in llvm_versions: - container_folder = f".devcontainer/{os_name}_{os_version}_{llvm_version}" + container_folder = ( + f".devcontainer/{os_name}_{os_version}_{llvm_version}" + ) os.makedirs(container_folder, exist_ok=True) - template_args = {"LLVM_VERSION": llvm_version, - "OS_NAME": os_name, "OS_VERSION": os_version} + template_args = { + "LLVM_VERSION": llvm_version, + "OS_NAME": os_name, + "OS_VERSION": os_version, + } renderer = pystache.Renderer(missing_tags="strict") - for template in ['devcontainer.json', "Dockerfile"]: + for template in ["devcontainer.json", "Dockerfile"]: template_name = f"{template_folder}/{template}.mustache" result_filename = f"{container_folder}/{template}" with open(template_name, "r") as t: @@ -50,41 +68,48 @@ def gh_workflows(args): template_folder = f"infrastructure/templates/github-actions/" workflow_folder = f".github/workflows/" - strategies = [] - for os_version in sorted(SUPPORTED_PLATFORMS["ubuntu"].keys()): - for llvm_version in SUPPORTED_PLATFORMS["ubuntu"][os_version]: - arg = {"OS_NAME": "ubuntu", "OS_VERSION": os_version, - "LLVM_VERSION": llvm_version} - strategies.append(arg) - - template_args = {"strategy": strategies, "OS_NAME": "Ubuntu"} - renderer = pystache.Renderer(missing_tags="strict") - for template in ['ci-ubuntu.yml']: - template_name = f"{template_folder}/{template}.mustache" - result_filename = f"{workflow_folder}/{template}" - with open(template_name, "r") as t: - result = renderer.render(t.read(), template_args) - with open(result_filename, "w") as f: - f.write(result) + for os_name in SUPPORTED_PLATFORMS.keys(): + strategies = [] + for os_version in sorted(SUPPORTED_PLATFORMS[os_name].keys()): + for llvm_version in SUPPORTED_PLATFORMS[os_name][os_version]: + arg = { + "OS_NAME": os_name, + "OS_VERSION": os_version, + "LLVM_VERSION": llvm_version, + } + strategies.append(arg) + + template_args = {"strategy": strategies, "OS_NAME": os_name.capitalize()} + renderer = pystache.Renderer(missing_tags="strict") + for template in [f"ci-{os_name}.yml"]: + template_name = f"{template_folder}/{template}.mustache" + result_filename = f"{workflow_folder}/{template}" + with open(template_name, "r") as t: + result = renderer.render(t.read(), template_args) + with open(result_filename, "w") as f: + f.write(result) def main(): parser = argparse.ArgumentParser( - prog='generator', description='Generates various infra files for Mull') + prog="generator", description="Generates various infra files for Mull" + ) subparsers = parser.add_subparsers( - title='subcommands', help='available subcommands', dest="cmd") - - parser_cmake = subparsers.add_parser( - 'cmake', help='Generates CMake preset file') - parser_cmake.add_argument('--os', choices=('ubuntu',), - help='Select OS for which to generate preset', type=str) + title="subcommands", help="available subcommands", dest="cmd" + ) + + parser_cmake = subparsers.add_parser("cmake", help="Generates CMake preset file") + parser_cmake.add_argument( + "--os", + choices=("ubuntu", "macos"), + help="Select OS for which to generate preset", + type=str, + ) parser_cmake.add_argument("--llvm_version", type=int) - subparsers.add_parser('devcontainers', help='Generates devcontainer files') - subparsers.add_parser('github_workflows', - help='Generates GitHub workflow files') - subparsers.add_parser('all', - help='Combines devcontainers and github_workflows') + subparsers.add_parser("devcontainers", help="Generates devcontainer files") + subparsers.add_parser("github_workflows", help="Generates GitHub workflow files") + subparsers.add_parser("all", help="Combines devcontainers and github_workflows") args = parser.parse_args() if args.cmd == "cmake": diff --git a/infrastructure/templates/cmake-presets/ubuntu.mustache b/infrastructure/templates/cmake-presets/CMakePresets.json.mustache similarity index 82% rename from infrastructure/templates/cmake-presets/ubuntu.mustache rename to infrastructure/templates/cmake-presets/CMakePresets.json.mustache index c159f7326..ad1f8422d 100644 --- a/infrastructure/templates/cmake-presets/ubuntu.mustache +++ b/infrastructure/templates/cmake-presets/CMakePresets.json.mustache @@ -4,12 +4,12 @@ { "name": "mull", "generator": "Ninja", - "binaryDir": "${sourceDir}/build.{{{LLVM_VERSION}}}.dir", + "binaryDir": "${sourceDir}/build.{{OS_NAME}}.{{{LLVM_VERSION}}}.dir", "cacheVariables": { "CMAKE_BUILD_TYPE": "Release", - "CMAKE_C_COMPILER": "clang-{{LLVM_VERSION}}", - "CMAKE_CXX_COMPILER": "clang++-{{LLVM_VERSION}}", - "CMAKE_PREFIX_PATH": "/usr/lib/llvm-{{LLVM_VERSION}}/cmake/;/usr/lib/cmake/clang-{{LLVM_VERSION}}/" + "CMAKE_C_COMPILER": "{{CC}}", + "CMAKE_CXX_COMPILER": "{{CXX}}", + "CMAKE_PREFIX_PATH": "{{CMAKE_PREFIX_PATH}}" } } ], diff --git a/infrastructure/templates/github-actions/ci-macos.yml.mustache b/infrastructure/templates/github-actions/ci-macos.yml.mustache new file mode 100644 index 000000000..5960630aa --- /dev/null +++ b/infrastructure/templates/github-actions/ci-macos.yml.mustache @@ -0,0 +1,64 @@ +{{=[[ ]]=}}name: macOS CI + +env: + CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +on: + push: + branches: ["main"] + tags: ["**"] + pull_request: + branches: ["main"] + +jobs: + macos: + name: LLVM ${{ matrix.LLVM_VERSION }} + runs-on: macos-latest + strategy: + matrix: + include: + [[#strategy]] + - OS_NAME: "[[ OS_NAME ]]" + LLVM_VERSION: [[ LLVM_VERSION ]] + [[/strategy]] + steps: + - name: Fix git + run: | + git config --global --add safe.directory $PWD + - uses: actions/checkout@v4 + with: + submodules: true + - uses: ./actions/detect-package-metadata + id: metadata + + - name: Run CI task + run: | + brew install zlib ncurses sqlite ninja llvm@${{ matrix.LLVM_VERSION }} + pip3 install -r requirements.txt --break-system-packages + ./infrastructure/generator.py cmake --os macos --llvm_version ${{ matrix.LLVM_VERSION }} + cmake --preset mull -DMULL_VERSION=${{ steps.metadata.outputs.version }}${{ steps.metadata.outputs.suffix }} + cmake --workflow --preset build-and-test + cmake --workflow --preset package + ./end2end-tests/setup_end2end_tests.sh + ./end2end-tests/run_end2end_tests.sh macos ${{ matrix.LLVM_VERSION }} + mull-runner-${{ matrix.LLVM_VERSION }} --version + + - name: Publish package + if: env.CLOUDSMITH_API_KEY != null + run: | + cloudsmith push raw \ + --name mull-`echo ${{ matrix.LLVM_VERSION }} | awk -F. ' { print $1 } '` \ + --version ${{ steps.metadata.outputs.version }}${{ steps.metadata.outputs.suffix }} \ + mull-project/mull-${{ steps.metadata.outputs.channel }} \ + --tags macos \ + build.${{ matrix.OS_NAME }}.${{ matrix.LLVM_VERSION }}.dir/*.zip + + - name: Move package + run: | + mkdir -p /tmp/packages + mv build.${{ matrix.OS_NAME }}.${{ matrix.LLVM_VERSION }}.dir/*.zip /tmp/packages/ + + - uses: ./actions/attach-package + with: + GH_API_KEY: ${{ secrets.GITHUB_TOKEN }} diff --git a/infrastructure/templates/github-actions/ci-ubuntu.yml.mustache b/infrastructure/templates/github-actions/ci-ubuntu.yml.mustache index 2400a35df..4c4b2c9c6 100644 --- a/infrastructure/templates/github-actions/ci-ubuntu.yml.mustache +++ b/infrastructure/templates/github-actions/ci-ubuntu.yml.mustache @@ -41,7 +41,7 @@ jobs: cmake --preset mull -DMULL_VERSION=${{ steps.metadata.outputs.version }}${{ steps.metadata.outputs.suffix }} cmake --workflow --preset build-and-test cmake --workflow --preset package - ./end2end-tests/run_end2end_tests.sh ${{ matrix.LLVM_VERSION }} + ./end2end-tests/run_end2end_tests.sh ${{ matrix.OS_NAME }} ${{ matrix.LLVM_VERSION }} mull-runner-${{ matrix.LLVM_VERSION }} --version - name: Publish package @@ -55,12 +55,12 @@ jobs: UBUNTU_CODENAME=$(. /etc/os-release; echo "${UBUNTU_CODENAME/*, /}") env CLOUDSMITH_API_KEY=${{ secrets.CLOUDSMITH_API_KEY }} cloudsmith push deb \ mull-project/mull-${{ steps.metadata.outputs.channel }}/${{ matrix.OS_NAME }}/${UBUNTU_CODENAME} \ - build.${{ matrix.LLVM_VERSION }}.dir/*.deb + build.${{ matrix.OS_NAME }}.${{ matrix.LLVM_VERSION }}.dir/*.deb - name: Move package run: | mkdir -p /tmp/packages - sudo mv build.${{ matrix.LLVM_VERSION }}.dir/*.deb /tmp/packages/ + sudo mv build.${{ matrix.OS_NAME }}.${{ matrix.LLVM_VERSION }}.dir/*.deb /tmp/packages/ - uses: ./actions/attach-package with: