Remove ansible from macOS CI

AlexDenisov · AlexDenisov · commit ac77ee4a07ed · 2024-12-15T11:50:24.000+01:00
diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml
@@ -5,49 +5,52 @@ env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 on:
-  pull_request:
   push:
     branches: ["main"]
     tags: ["**"]
+  pull_request:
+    branches: ["main"]
 
 jobs:
   macos:
     name: LLVM ${{ matrix.LLVM_VERSION }}
     runs-on: macos-latest
     strategy:
       matrix:
-        LLVM_VERSION: [13, 14, 15, 16, 17, 18]
-
+        include:
+          - OS_NAME: "macos"
+            LLVM_VERSION: 13
+          - OS_NAME: "macos"
+            LLVM_VERSION: 14
+          - OS_NAME: "macos"
+            LLVM_VERSION: 15
+          - OS_NAME: "macos"
+            LLVM_VERSION: 16
+          - OS_NAME: "macos"
+            LLVM_VERSION: 17
+          - OS_NAME: "macos"
+            LLVM_VERSION: 18
     steps:
-      - name: Debugging
-        run: |
-          env | sort
-          cat $GITHUB_EVENT_PATH
-
       - name: Fix git
         run: |
           git config --global --add safe.directory $PWD
-
-      - name: Checkout sources
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
-
       - uses: ./actions/detect-package-metadata
         id: metadata
 
       - name: Run CI task
         run: |
-          brew install ansible
-          cd infrastructure && \
-          ansible-playbook macos-playbook.yaml \
-            -e llvm_version="${{ matrix.LLVM_VERSION }}" \
-            -e source_dir=$PWD/.. \
-            -e gitref=$GITHUB_SHA \
-            -e host=localhost \
-            -e SDKROOT=`xcrun -show-sdk-path` \
-            -e mull_version=${{ steps.metadata.outputs.version }}${{ steps.metadata.outputs.suffix }} \
-            --verbose
+          brew install zlib ncurses sqlite ninja llvm@${{ matrix.LLVM_VERSION }}
+          pip3 install -r requirements.txt --break-system-packages
+          ./infrastructure/generator.py cmake --os macos --llvm_version ${{ matrix.LLVM_VERSION }}
+          cmake --preset mull -DMULL_VERSION=${{ steps.metadata.outputs.version }}${{ steps.metadata.outputs.suffix }}
+          cmake --workflow --preset build-and-test
+          cmake --workflow --preset package
+          ./end2end-tests/setup_end2end_tests.sh
+          ./end2end-tests/run_end2end_tests.sh macos ${{ matrix.LLVM_VERSION }}
+          mull-runner-${{ matrix.LLVM_VERSION }} --version
 
       - name: Publish package
         if: env.CLOUDSMITH_API_KEY != null
@@ -57,12 +60,12 @@ jobs:
             --version ${{ steps.metadata.outputs.version }}${{ steps.metadata.outputs.suffix }} \
             mull-project/mull-${{ steps.metadata.outputs.channel }} \
             --tags macos \
-            infrastructure/packages/`cat infrastructure/PACKAGE_FILE_NAME`.zip
+            build.${{ matrix.OS_NAME }}.${{ matrix.LLVM_VERSION }}.dir/*.zip
 
       - name: Move package
         run: |
           mkdir -p /tmp/packages
-          mv infrastructure/packages/`cat infrastructure/PACKAGE_FILE_NAME`.zip /tmp/packages/`cat infrastructure/PACKAGE_FILE_NAME`.zip
+          mv build.${{ matrix.OS_NAME }}.${{ matrix.LLVM_VERSION }}.dir/*.zip /tmp/packages/
 
       - uses: ./actions/attach-package
         with:
diff --git a/.github/workflows/ci-ubuntu.yml b/.github/workflows/ci-ubuntu.yml
@@ -63,7 +63,7 @@ jobs:
             cmake --preset mull -DMULL_VERSION=${{ steps.metadata.outputs.version }}${{ steps.metadata.outputs.suffix }}
             cmake --workflow --preset build-and-test
             cmake --workflow --preset package
-            ./end2end-tests/run_end2end_tests.sh ${{ matrix.LLVM_VERSION }}
+            ./end2end-tests/run_end2end_tests.sh ${{ matrix.OS_NAME }} ${{ matrix.LLVM_VERSION }}
             mull-runner-${{ matrix.LLVM_VERSION }} --version
 
       - name: Publish package
@@ -77,12 +77,12 @@ jobs:
             UBUNTU_CODENAME=$(. /etc/os-release; echo "${UBUNTU_CODENAME/*, /}")
             env CLOUDSMITH_API_KEY=${{ secrets.CLOUDSMITH_API_KEY }} cloudsmith push deb \
               mull-project/mull-${{ steps.metadata.outputs.channel }}/${{ matrix.OS_NAME }}/${UBUNTU_CODENAME} \
-              build.${{ matrix.LLVM_VERSION }}.dir/*.deb
+              build.${{ matrix.OS_NAME }}.${{ matrix.LLVM_VERSION }}.dir/*.deb
 
       - name: Move package
         run: |
           mkdir -p /tmp/packages
-          sudo mv build.${{ matrix.LLVM_VERSION }}.dir/*.deb /tmp/packages/
+          sudo mv build.${{ matrix.OS_NAME }}.${{ matrix.LLVM_VERSION }}.dir/*.deb /tmp/packages/
 
       - uses: ./actions/attach-package
         with:
diff --git a/end2end-tests/run_end2end_tests.sh b/end2end-tests/run_end2end_tests.sh
@@ -1,16 +1,30 @@
-#!/usr/bin/bash
+#!/bin/bash
 
 set -e
 set -x
 
-LLVM_VERSION=$1
+OS_NAME=$1
+LLVM_VERSION=$2
+BUILD_DIR=build.$OS_NAME.$LLVM_VERSION.dir
 
 if (( $LLVM_VERSION < 16 )); then
   EXTRA_FLAGS=-fexperimental-new-pass-manager
 fi
 
-# Install mull system-wide
-dpkg -i build.${LLVM_VERSION}.dir/*.deb
+if [ "$OS_NAME" == "macos" ]
+then
+  sudo cmake --install $BUILD_DIR
+  PLUGIN_PATH=/usr/local/lib/mull-ir-frontend-$LLVM_VERSION
+  MULL_CC=/opt/homebrew/opt/llvm@$LLVM_VERSION/bin/clang
+  MULL_CXX=/opt/homebrew/opt/llvm@$LLVM_VERSION/bin/clang++
+else
+  dpkg -i $BUILD_DIR/*.deb
+  PLUGIN_PATH=/usr/lib/mull-ir-frontend-$LLVM_VERSION
+  MULL_CC=clang-$LLVM_VERSION
+  MULL_CXX=clang++-$LLVM_VERSION
+fi
+
+which mull-runner-$LLVM_VERSION
 
 pushd `dirname "$0"`
 
@@ -23,10 +37,10 @@ function test_fmt() {
   pushd fmt.build.dir
 
   cmake -G Ninja \
-    -DCMAKE_CXX_FLAGS="-grecord-command-line -fpass-plugin=/usr/lib/mull-ir-frontend-$LLVM_VERSION $EXTRA_FLAGS" \
+    -DCMAKE_CXX_FLAGS="-grecord-command-line -fpass-plugin=$PLUGIN_PATH $EXTRA_FLAGS" \
     -DCMAKE_BUILD_TYPE=Debug \
-    -DCMAKE_C_COMPILER=clang-$LLVM_VERSION \
-    -DCMAKE_CXX_COMPILER=clang++-$LLVM_VERSION \
+    -DCMAKE_C_COMPILER=$MULL_CC \
+    -DCMAKE_CXX_COMPILER=$MULL_CXX \
     ..
 
   ninja core-test
@@ -42,13 +56,19 @@ function test_fmt() {
 function test_openssl() {
   pushd openssl
 
-  env CC=clang-$LLVM_VERSION ./config -g -O0 \
+  env CC=$MULL_CC ./config -g -O0 \
      $EXTRA_FLAGS \
     -grecord-command-line \
-    -fpass-plugin=/usr/lib/mull-ir-frontend-$LLVM_VERSION
+    -fpass-plugin=$PLUGIN_PATH
 
   make build_generated -j
   make ./test/bio_enc_test -j
+
+  if [ "$OS_NAME" == "macos" ]
+  then
+    install_name_tool -change /usr/local/lib/libcrypto.3.dylib $PWD/libcrypto.3.dylib ./test/bio_enc_test
+  fi
+
   env LD_LIBRARY_PATH=. ./test/bio_enc_test
   env LD_LIBRARY_PATH=. mull-runner-$LLVM_VERSION --allow-surviving ./test/bio_enc_test
 
diff --git a/end2end-tests/setup_end2end_tests.sh b/end2end-tests/setup_end2end_tests.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/bash
+#!/bin/bash
 
 set -e
 set -x
diff --git a/infrastructure/generator.py b/infrastructure/generator.py
@@ -6,38 +6,56 @@
 import shutil
 
 SUPPORTED_PLATFORMS = {
-    "ubuntu": {
-        "20.04": [12],
-        "22.04": [13, 14, 15],
-        "24.04": [14, 15, 16, 17, 18]
-    }
+    "ubuntu": {"20.04": [12], "22.04": [13, 14, 15], "24.04": [14, 15, 16, 17, 18]},
+    "macos": {"latest": [13, 14, 15, 16, 17, 18]},
 }
 
 
 def cmake(args):
-    template_name = f"infrastructure/templates/cmake-presets/{args.os}.mustache"
-    template_args = {"LLVM_VERSION": args.llvm_version}
+    os_specific_args = {
+        "ubuntu": {
+            "CC": f"clang-{args.llvm_version}",
+            "CXX": f"clang++-{args.llvm_version}",
+            "CMAKE_PREFIX_PATH": f"/usr/lib/llvm-{args.llvm_version}/cmake/;/usr/lib/cmake/clang-{args.llvm_version}/",
+        },
+        "macos": {
+            "CC": f"/opt/homebrew/opt/llvm@{args.llvm_version}/bin/clang",
+            "CXX": f"/opt/homebrew/opt/llvm@{args.llvm_version}/bin/clang++",
+            "CMAKE_PREFIX_PATH": f"/opt/homebrew/opt/llvm@{args.llvm_version}/lib/cmake/llvm/;/opt/homebrew/opt/llvm@{args.llvm_version}/lib/cmake/clang/",
+        },
+    }
+    template_name = f"infrastructure/templates/cmake-presets/CMakePresets.json.mustache"
+    template_args = os_specific_args[args.os]
+    template_args["LLVM_VERSION"] = args.llvm_version
+    template_args["OS_NAME"] = args.os
     renderer = pystache.Renderer(missing_tags="strict")
     with open(template_name, "r") as t:
         result = renderer.render(t.read(), template_args)
-        with open('CMakePresets.json', "w") as f:
+        with open("CMakePresets.json", "w") as f:
             f.write(result)
 
 
 def devcontainers(args):
     shutil.rmtree(".devcontainer")
-    for (os_name, platform) in SUPPORTED_PLATFORMS.items():
+    for os_name, platform in SUPPORTED_PLATFORMS.items():
+        if os_name.lower() == "macos":
+            continue
         template_folder = f"infrastructure/templates/devcontainers/{os_name}"
-        for (os_version, llvm_versions) in platform.items():
+        for os_version, llvm_versions in platform.items():
             for llvm_version in llvm_versions:
-                container_folder = f".devcontainer/{os_name}_{os_version}_{llvm_version}"
+                container_folder = (
+                    f".devcontainer/{os_name}_{os_version}_{llvm_version}"
+                )
                 os.makedirs(container_folder, exist_ok=True)
 
-                template_args = {"LLVM_VERSION": llvm_version,
-                                 "OS_NAME": os_name, "OS_VERSION": os_version}
+                template_args = {
+                    "LLVM_VERSION": llvm_version,
+                    "OS_NAME": os_name,
+                    "OS_VERSION": os_version,
+                }
 
                 renderer = pystache.Renderer(missing_tags="strict")
-                for template in ['devcontainer.json', "Dockerfile"]:
+                for template in ["devcontainer.json", "Dockerfile"]:
                     template_name = f"{template_folder}/{template}.mustache"
                     result_filename = f"{container_folder}/{template}"
                     with open(template_name, "r") as t:
@@ -50,41 +68,48 @@ def gh_workflows(args):
     template_folder = f"infrastructure/templates/github-actions/"
     workflow_folder = f".github/workflows/"
 
-    strategies = []
-    for os_version in sorted(SUPPORTED_PLATFORMS["ubuntu"].keys()):
-        for llvm_version in SUPPORTED_PLATFORMS["ubuntu"][os_version]:
-            arg = {"OS_NAME": "ubuntu", "OS_VERSION": os_version,
-                   "LLVM_VERSION": llvm_version}
-            strategies.append(arg)
-
-    template_args = {"strategy": strategies, "OS_NAME": "Ubuntu"}
-    renderer = pystache.Renderer(missing_tags="strict")
-    for template in ['ci-ubuntu.yml']:
-        template_name = f"{template_folder}/{template}.mustache"
-        result_filename = f"{workflow_folder}/{template}"
-        with open(template_name, "r") as t:
-            result = renderer.render(t.read(), template_args)
-            with open(result_filename, "w") as f:
-                f.write(result)
+    for os_name in SUPPORTED_PLATFORMS.keys():
+        strategies = []
+        for os_version in sorted(SUPPORTED_PLATFORMS[os_name].keys()):
+            for llvm_version in SUPPORTED_PLATFORMS[os_name][os_version]:
+                arg = {
+                    "OS_NAME": os_name,
+                    "OS_VERSION": os_version,
+                    "LLVM_VERSION": llvm_version,
+                }
+                strategies.append(arg)
+
+        template_args = {"strategy": strategies, "OS_NAME": os_name.capitalize()}
+        renderer = pystache.Renderer(missing_tags="strict")
+        for template in [f"ci-{os_name}.yml"]:
+            template_name = f"{template_folder}/{template}.mustache"
+            result_filename = f"{workflow_folder}/{template}"
+            with open(template_name, "r") as t:
+                result = renderer.render(t.read(), template_args)
+                with open(result_filename, "w") as f:
+                    f.write(result)
 
 
 def main():
     parser = argparse.ArgumentParser(
-        prog='generator', description='Generates various infra files for Mull')
+        prog="generator", description="Generates various infra files for Mull"
+    )
     subparsers = parser.add_subparsers(
-        title='subcommands', help='available subcommands', dest="cmd")
-
-    parser_cmake = subparsers.add_parser(
-        'cmake', help='Generates CMake preset file')
-    parser_cmake.add_argument('--os', choices=('ubuntu',),
-                              help='Select OS for which to generate preset', type=str)
+        title="subcommands", help="available subcommands", dest="cmd"
+    )
+
+    parser_cmake = subparsers.add_parser("cmake", help="Generates CMake preset file")
+    parser_cmake.add_argument(
+        "--os",
+        choices=("ubuntu", "macos"),
+        help="Select OS for which to generate preset",
+        type=str,
+    )
     parser_cmake.add_argument("--llvm_version", type=int)
 
-    subparsers.add_parser('devcontainers', help='Generates devcontainer files')
-    subparsers.add_parser('github_workflows',
-                          help='Generates GitHub workflow files')
-    subparsers.add_parser('all',
-                          help='Combines devcontainers and github_workflows')
+    subparsers.add_parser("devcontainers", help="Generates devcontainer files")
+    subparsers.add_parser("github_workflows", help="Generates GitHub workflow files")
+    subparsers.add_parser("all", help="Combines devcontainers and github_workflows")
 
     args = parser.parse_args()
     if args.cmd == "cmake":
diff --git a/infrastructure/templates/cmake-presets/CMakePresets.json.mustache b/infrastructure/templates/cmake-presets/CMakePresets.json.mustache
@@ -4,12 +4,12 @@
         {
             "name": "mull",
             "generator": "Ninja",
-            "binaryDir": "${sourceDir}/build.{{{LLVM_VERSION}}}.dir",
+            "binaryDir": "${sourceDir}/build.{{OS_NAME}}.{{{LLVM_VERSION}}}.dir",
             "cacheVariables": {
                 "CMAKE_BUILD_TYPE": "Release",
-                "CMAKE_C_COMPILER": "clang-{{LLVM_VERSION}}",
-                "CMAKE_CXX_COMPILER": "clang++-{{LLVM_VERSION}}",
-                "CMAKE_PREFIX_PATH": "/usr/lib/llvm-{{LLVM_VERSION}}/cmake/;/usr/lib/cmake/clang-{{LLVM_VERSION}}/"
+                "CMAKE_C_COMPILER": "{{CC}}",
+                "CMAKE_CXX_COMPILER": "{{CXX}}",
+                "CMAKE_PREFIX_PATH": "{{CMAKE_PREFIX_PATH}}"
             }
         }
     ],
diff --git a/infrastructure/templates/github-actions/ci-macos.yml.mustache b/infrastructure/templates/github-actions/ci-macos.yml.mustache
diff --git a/infrastructure/templates/github-actions/ci-ubuntu.yml.mustache b/infrastructure/templates/github-actions/ci-ubuntu.yml.mustache

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-#!/usr/bin/bash`
	`1`	`+#!/bin/bash`
`2`	`2`
`3`	`3`	`set -e`
`4`	`4`	`set -x`
Original file line number	Diff line number	Diff line change
`@@ -4,12 +4,12 @@`
`4`	`4`	`{`
`5`	`5`	`"name": "mull",`
`6`	`6`	`"generator": "Ninja",`
`7`		`- "binaryDir": "${sourceDir}/build.{{{LLVM_VERSION}}}.dir",`
	`7`	`+ "binaryDir": "${sourceDir}/build.{{OS_NAME}}.{{{LLVM_VERSION}}}.dir",`
`8`	`8`	`"cacheVariables": {`
`9`	`9`	`"CMAKE_BUILD_TYPE": "Release",`
`10`		`- "CMAKE_C_COMPILER": "clang-{{LLVM_VERSION}}",`
`11`		`- "CMAKE_CXX_COMPILER": "clang++-{{LLVM_VERSION}}",`
`12`		`- "CMAKE_PREFIX_PATH": "/usr/lib/llvm-{{LLVM_VERSION}}/cmake/;/usr/lib/cmake/clang-{{LLVM_VERSION}}/"`
	`10`	`+ "CMAKE_C_COMPILER": "{{CC}}",`
	`11`	`+ "CMAKE_CXX_COMPILER": "{{CXX}}",`
	`12`	`+ "CMAKE_PREFIX_PATH": "{{CMAKE_PREFIX_PATH}}"`
`13`	`13`	`}`
`14`	`14`	`}`
`15`	`15`	`],`