From d839b3fb10553ff829b4e6ad5858aed127ec41af Mon Sep 17 00:00:00 2001 From: Boris Staletic Date: Fri, 5 Apr 2024 09:18:28 +0200 Subject: [PATCH 01/13] Try to enable openmp --- package_llvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package_llvm.py b/package_llvm.py index a7feef3..02ec3b0 100755 --- a/package_llvm.py +++ b/package_llvm.py @@ -205,7 +205,7 @@ def BuildLlvm( build_dir, # A release build implies LLVM_ENABLE_ASSERTIONS=OFF. '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_INSTALL_PREFIX={}'.format( install_dir ), - '-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra', + '-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra;openmp', '-DLLVM_DEFAULT_TARGET_TRIPLE={}'.format( target ), '-DLLVM_TARGETS_TO_BUILD=all', '-DLLVM_TABLEGEN={}'.format( From 930d276f6ade04b4ad865806866f371f13f7ef3d Mon Sep 17 00:00:00 2001 From: Boris Staletic Date: Fri, 5 Apr 2024 16:43:57 +0200 Subject: [PATCH 02/13] Update macOS runner configuration This should avoid the need to cross compile for macOS, though M1 runners have only 7GB of RAM. --- .github/workflows/package_llvm.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/package_llvm.yml b/.github/workflows/package_llvm.yml index fba4e08..301b396 100644 --- a/.github/workflows/package_llvm.yml +++ b/.github/workflows/package_llvm.yml @@ -30,12 +30,10 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: ./package_llvm.py --target-architecture ${{ matrix.target }} ${{ github.event.inputs.version }} - build-macos: - runs-on: macos-11 + build-macos-x86: + runs-on: macos-13 strategy: fail-fast: false - matrix: - target: [ x86_64, arm64 ] steps: - uses: actions/checkout@v4 - name: install requirements @@ -44,4 +42,18 @@ jobs: env: GITHUB_USERNAME: ${{ github.actor }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: ./package_llvm.py --target-architecture ${{ matrix.target }} ${{ github.event.inputs.version }} + run: ./package_llvm.py --target-architecture x86_64 ${{ github.event.inputs.version }} + + build-macos-arm64: + runs-on: macos-14 + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v4 + - name: install requirements + run: pip3 install -r requirements.txt + - name: Package + env: + GITHUB_USERNAME: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: ./package_llvm.py --target-architecture arm64 ${{ github.event.inputs.version }} From 783f2b96b6b1474369c70ac084e26c1d5d6d3367 Mon Sep 17 00:00:00 2001 From: Boris Staletic Date: Fri, 5 Apr 2024 17:23:22 +0200 Subject: [PATCH 03/13] Force pip to install packages anyway --- .github/workflows/package_llvm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package_llvm.yml b/.github/workflows/package_llvm.yml index 301b396..06e01fb 100644 --- a/.github/workflows/package_llvm.yml +++ b/.github/workflows/package_llvm.yml @@ -51,7 +51,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: install requirements - run: pip3 install -r requirements.txt + run: pip3 install --break-system-packages -r requirements.txt - name: Package env: GITHUB_USERNAME: ${{ github.actor }} From c676b4ef5be8e07989a145be348b49d21f78a3cd Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Fri, 5 Apr 2024 21:07:19 +0100 Subject: [PATCH 04/13] Add --no-upload option for local testing of package_llvm.py --- package_llvm.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/package_llvm.py b/package_llvm.py index 02ec3b0..3f9688d 100755 --- a/package_llvm.py +++ b/package_llvm.py @@ -395,6 +395,9 @@ def ParseArguments(): parser.add_argument( '--release-candidate', type = int, help = 'LLVM release candidate number.' ) + parser.add_argument( '--no-upload', action = 'store_true', + help = "Don't upload the archive to GitHub." ) + parser.add_argument( '--gh-user', action='store', help = 'GitHub user name. Defaults to environment ' 'variable: GITHUB_USERNAME' ) @@ -416,17 +419,18 @@ def ParseArguments(): args = parser.parse_args() - if not args.gh_user: - if 'GITHUB_USERNAME' not in os.environ: - sys.exit( 'ERROR: Must specify either --gh-user or ' - 'GITHUB_USERNAME in environment' ) - args.gh_user = os.environ[ 'GITHUB_USERNAME' ] + if not args.no_upload: + if not args.gh_user: + if 'GITHUB_USERNAME' not in os.environ: + sys.exit( 'ERROR: Must specify either --gh-user or ' + 'GITHUB_USERNAME in environment' ) + args.gh_user = os.environ[ 'GITHUB_USERNAME' ] - if not args.gh_token: - if 'GITHUB_TOKEN' not in os.environ: - sys.exit( 'ERROR: Must specify either --gh-token or ' - 'GITHUB_TOKEN in environment' ) - args.gh_token = os.environ[ 'GITHUB_TOKEN' ] + if not args.gh_token: + if 'GITHUB_TOKEN' not in os.environ: + sys.exit( 'ERROR: Must specify either --gh-token or ' + 'GITHUB_TOKEN in environment' ) + args.gh_token = os.environ[ 'GITHUB_TOKEN' ] return args @@ -478,7 +482,9 @@ def Main(): if not os.path.exists( bundle_path ): with WorkingDirectory( base_dir ): BundleLlvm( bundle_name, archive_name, llvm_install_dir, bundle_version ) - UploadLlvm( args, bundle_path ) + + if not args.no_upload: + UploadLlvm( args, bundle_path ) if __name__ == "__main__": From ae1139c93759936a9abeb94f44dbbccc1018f135 Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Fri, 5 Apr 2024 21:07:36 +0100 Subject: [PATCH 05/13] Remove unused matrix build options --- .github/workflows/package_llvm.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/package_llvm.yml b/.github/workflows/package_llvm.yml index 06e01fb..0df18d2 100644 --- a/.github/workflows/package_llvm.yml +++ b/.github/workflows/package_llvm.yml @@ -32,8 +32,6 @@ jobs: build-macos-x86: runs-on: macos-13 - strategy: - fail-fast: false steps: - uses: actions/checkout@v4 - name: install requirements @@ -46,8 +44,6 @@ jobs: build-macos-arm64: runs-on: macos-14 - strategy: - fail-fast: false steps: - uses: actions/checkout@v4 - name: install requirements From 850b5dd43fd97336ab365054131b14b3158bc498 Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Fri, 5 Apr 2024 21:18:24 +0100 Subject: [PATCH 06/13] Use Ninja --- package_llvm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/package_llvm.py b/package_llvm.py index 3f9688d..6e10638 100755 --- a/package_llvm.py +++ b/package_llvm.py @@ -202,6 +202,7 @@ def BuildLlvm( build_dir, # variables defined by LLVM. cmake_configure_args = [ cmake, + '-G', 'Ninja', # A release build implies LLVM_ENABLE_ASSERTIONS=OFF. '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_INSTALL_PREFIX={}'.format( install_dir ), @@ -246,6 +247,7 @@ def BuildTableGen( build_dir, llvm_source_dir ): cmake = shutil.which( 'cmake' ) subprocess.check_call( [ cmake, + '-G', 'Ninja', '-DCMAKE_BUILD_TYPE=Release', '-DLLVM_ENABLE_PROJECTS=clang', os.path.join( llvm_source_dir, 'llvm' ) ] ) From 25c57b329d05aa907db8db78c47cea4313f53ca2 Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Fri, 5 Apr 2024 21:18:39 +0100 Subject: [PATCH 07/13] LLVM_INCLUDE_GO_TESTS is no longer used? --- package_llvm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/package_llvm.py b/package_llvm.py index 6e10638..a3f8211 100755 --- a/package_llvm.py +++ b/package_llvm.py @@ -216,7 +216,6 @@ def BuildLlvm( build_dir, '-DLLVM_TARGET_ARCH={}'.format( target_architecture ), '-DLLVM_INCLUDE_EXAMPLES=OFF', '-DLLVM_INCLUDE_TESTS=OFF', - '-DLLVM_INCLUDE_GO_TESTS=OFF', '-DLLVM_INCLUDE_DOCS=OFF', '-DLLVM_ENABLE_TERMINFO=OFF', '-DLLVM_ENABLE_ZLIB=OFF', From 9e627c8fb84e2919b880c2081755061f7e43fb2d Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Fri, 5 Apr 2024 21:42:27 +0100 Subject: [PATCH 08/13] Use ccache if available --- package_llvm.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/package_llvm.py b/package_llvm.py index a3f8211..082155f 100755 --- a/package_llvm.py +++ b/package_llvm.py @@ -187,6 +187,15 @@ def GetLogicalCores(): return subprocess.check_output( cmd ).decode( 'utf-8' ).strip() +def GetCacheArgs( build_dir ): + return [ + '-DLLVM_CCACHE_BUILD=ON', + '-DLLVM_CCACHE_MAXSIZE=5G', + '-DLLVM_CCACHE_DIR={}'.format( + os.path.abspath( os.path.join( build_dir, '..', 'ccache' ) ) ), + ] + + def BuildLlvm( build_dir, install_dir, llvm_source_dir, @@ -222,6 +231,7 @@ def BuildLlvm( build_dir, '-DLLVM_ENABLE_LIBEDIT=OFF', '-DLLVM_ENABLE_LIBXML2=OFF', '-DLLVM_ENABLE_ZSTD=OFF', + *GetCacheArgs( build_dir ), os.path.join( llvm_source_dir, 'llvm' ) ] if target != host: # We're cross compilinging and need a toolchain file. @@ -249,6 +259,7 @@ def BuildTableGen( build_dir, llvm_source_dir ): '-G', 'Ninja', '-DCMAKE_BUILD_TYPE=Release', '-DLLVM_ENABLE_PROJECTS=clang', + *GetCacheArgs( build_dir ), os.path.join( llvm_source_dir, 'llvm' ) ] ) subprocess.check_call( [ From 1361d35210ce091a421f71eebe199c1b8b7452c1 Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Fri, 5 Apr 2024 22:56:36 +0100 Subject: [PATCH 09/13] Fix cross compiling on macs This does 2 things: 1. When compiling assembly files (.S), specify the cross-compilation flags. This fixes the build of openmp, which has a cute little asm file, curiously named "Linux" but applies to Unix and Windows too. 2. Makes compiling x86_64 from an arm64 mac actually work, rather than just silently building arm64 objects and putting them in a directory named x86_64. --- package_llvm.py | 6 ++++-- toolchain_files/arm64-apple-darwin.cmake | 1 + toolchain_files/x86_64-apple-darwin.cmake | 6 ++++++ 3 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 toolchain_files/x86_64-apple-darwin.cmake diff --git a/package_llvm.py b/package_llvm.py index 082155f..a3f8803 100755 --- a/package_llvm.py +++ b/package_llvm.py @@ -59,12 +59,14 @@ }, 'Darwin': { 'x86_64': { - 'host': 'x86_64-apple-darwin', + 'host': 'x86_64-apple-darwin' + if platform.processor() != 'arm' else 'arm64-apple-darwin', 'target': 'x86_64-apple-darwin', 'archive': 'x86_64-apple-darwin' }, 'arm64': { - 'host': 'x86_64-apple-darwin', + 'host': 'x86_64-apple-darwin' + if platform.processor() != 'arm' else 'arm64-apple-darwin', 'target': 'arm64-apple-darwin', 'archive': 'arm64-apple-darwin' } diff --git a/toolchain_files/arm64-apple-darwin.cmake b/toolchain_files/arm64-apple-darwin.cmake index 0814a06..24acce6 100644 --- a/toolchain_files/arm64-apple-darwin.cmake +++ b/toolchain_files/arm64-apple-darwin.cmake @@ -3,3 +3,4 @@ set(CMAKE_SYSTEM_PROCESSOR arm64) set(CMAKE_C_FLAGS_INIT "-arch arm64") set(CMAKE_CXX_FLAGS_INIT "-arch arm64") +set(CMAKE_ASM_FLAGS_INIT "-arch arm64") diff --git a/toolchain_files/x86_64-apple-darwin.cmake b/toolchain_files/x86_64-apple-darwin.cmake new file mode 100644 index 0000000..db406d3 --- /dev/null +++ b/toolchain_files/x86_64-apple-darwin.cmake @@ -0,0 +1,6 @@ +set(CMAKE_SYSTEM_NAME Darwin) +set(CMAKE_SYSTEM_PROCESSOR x86_64) + +set(CMAKE_C_FLAGS_INIT "-arch x86_64") +set(CMAKE_CXX_FLAGS_INIT "-arch x86_64") +set(CMAKE_ASM_FLAGS_INIT "-arch x86_64") From d3fb439db72165fdf527154d61e301ea2231d3eb Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Fri, 5 Apr 2024 23:02:07 +0100 Subject: [PATCH 10/13] Go back to building on macos 13 now that cross compile works --- .github/workflows/package_llvm.yml | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/workflows/package_llvm.yml b/.github/workflows/package_llvm.yml index 0df18d2..23f8c1a 100644 --- a/.github/workflows/package_llvm.yml +++ b/.github/workflows/package_llvm.yml @@ -30,8 +30,12 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: ./package_llvm.py --target-architecture ${{ matrix.target }} ${{ github.event.inputs.version }} - build-macos-x86: + build-macos: runs-on: macos-13 + strategy: + fail-fast: false + matrix: + target: [ x86_64, arm64 ] steps: - uses: actions/checkout@v4 - name: install requirements @@ -40,16 +44,4 @@ jobs: env: GITHUB_USERNAME: ${{ github.actor }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: ./package_llvm.py --target-architecture x86_64 ${{ github.event.inputs.version }} - - build-macos-arm64: - runs-on: macos-14 - steps: - - uses: actions/checkout@v4 - - name: install requirements - run: pip3 install --break-system-packages -r requirements.txt - - name: Package - env: - GITHUB_USERNAME: ${{ github.actor }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: ./package_llvm.py --target-architecture arm64 ${{ github.event.inputs.version }} + run: ./package_llvm.py --target-architecture ${{ matrix.target }} ${{ github.event.inputs.version }} From d1ca904356dc2313aa222070778528187e5e1c9b Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Fri, 5 Apr 2024 23:02:26 +0100 Subject: [PATCH 11/13] Don't allow cross compile without toolchain file --- package_llvm.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/package_llvm.py b/package_llvm.py index a3f8803..39525eb 100755 --- a/package_llvm.py +++ b/package_llvm.py @@ -243,6 +243,13 @@ def BuildLlvm( build_dir, if os.path.exists( toolchain_file ): cmake_configure_args.append( '-DCMAKE_TOOLCHAIN_FILE={}'.format( toolchain_file ) ) + else: + print( "WARNING: Cross compiling, but no toolchain file found for " + + toolchain_file ) + sys.exit( 1 ) + + print( 'CMake Args: {}'.format( ' '.join( cmake_configure_args ) ) ) + subprocess.check_call( cmake_configure_args ) subprocess.check_call( [ From b0ba3f0dba06f578075aceae67c8e970c8febd24 Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Fri, 5 Apr 2024 23:13:45 +0100 Subject: [PATCH 12/13] Only use ninja if installed --- package_llvm.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/package_llvm.py b/package_llvm.py index 39525eb..170dd85 100755 --- a/package_llvm.py +++ b/package_llvm.py @@ -189,6 +189,10 @@ def GetLogicalCores(): return subprocess.check_output( cmd ).decode( 'utf-8' ).strip() +def GetGeneratorArgs(): + return shutil.which( 'ninja' ) and [ '-G', 'Ninja' ] or [] + + def GetCacheArgs( build_dir ): return [ '-DLLVM_CCACHE_BUILD=ON', @@ -213,7 +217,7 @@ def BuildLlvm( build_dir, # variables defined by LLVM. cmake_configure_args = [ cmake, - '-G', 'Ninja', + *GetGeneratorArgs(), # A release build implies LLVM_ENABLE_ASSERTIONS=OFF. '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_INSTALL_PREFIX={}'.format( install_dir ), @@ -265,7 +269,7 @@ def BuildTableGen( build_dir, llvm_source_dir ): cmake = shutil.which( 'cmake' ) subprocess.check_call( [ cmake, - '-G', 'Ninja', + *GetGeneratorArgs(), '-DCMAKE_BUILD_TYPE=Release', '-DLLVM_ENABLE_PROJECTS=clang', *GetCacheArgs( build_dir ), From 992bb655672beb3f8175ff9238f68840a8c36fa7 Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Fri, 5 Apr 2024 23:24:32 +0100 Subject: [PATCH 13/13] Despite the docs, you can only enable ccache if it exist --- package_llvm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package_llvm.py b/package_llvm.py index 170dd85..e40f4af 100755 --- a/package_llvm.py +++ b/package_llvm.py @@ -194,12 +194,12 @@ def GetGeneratorArgs(): def GetCacheArgs( build_dir ): - return [ + return shutil.which( 'ccache' ) and [ '-DLLVM_CCACHE_BUILD=ON', '-DLLVM_CCACHE_MAXSIZE=5G', '-DLLVM_CCACHE_DIR={}'.format( os.path.abspath( os.path.join( build_dir, '..', 'ccache' ) ) ), - ] + ] or [] def BuildLlvm( build_dir,