Skip to content

reduce time consumtion of unpacking LLVM cache items #805

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 27, 2021
Merged

reduce time consumtion of unpacking LLVM cache items #805

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 34 additions & 39 deletions .github/workflows/compilation_on_android_ubuntu_macos.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,11 @@ concurrency:
cancel-in-progress: true

env:
AOT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0"
AOT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0"
CLASSIC_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0"
FAST_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0"
JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0"
LAZY_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1"
FAST_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0"
JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0"
LAZY_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1"
LLVM_CACHE_SUFFIX: "build-llvm_libraries_ex"

jobs:
Expand Down Expand Up @@ -118,9 +118,13 @@ jobs:
if: ${{ matrix.light == 'green' }}
uses: actions/cache@v2
with:
path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
path: |
./core/deps/llvm/build/bin
./core/deps/llvm/build/include
./core/deps/llvm/build/lib
./core/deps/llvm/build/libexec
./core/deps/llvm/build/share
key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
restore-keys: ${{ matrix.os }}-${{env.LLVM_CACHE_SUFFIX }}

- name: Build llvm and clang from source on ubuntu
id: build_llvm_ubuntu
Expand All @@ -134,11 +138,6 @@ jobs:
run: /usr/bin/env python3 ./build_llvm.py --arch X86 WebAssembly
working-directory: build-scripts

- name: package LLVM
if: ${{ matrix.light == 'green' && (steps.build_llvm_ubuntu.conclusion == 'success' || steps.build_llvm_macos.conclusion == 'success')}}
run: mv LLVM-13.0.0-*.tar.gz LLVM-13.0.0.tar.gz
working-directory: core/deps/llvm/build/

build_wamrc:
needs: [build_llvm_libraries, check_repo]
runs-on: ${{ matrix.os }}
Expand All @@ -165,19 +164,18 @@ jobs:
if: ${{ matrix.light == 'green' }}
uses: actions/cache@v2
with:
path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
path: |
./core/deps/llvm/build/bin
./core/deps/llvm/build/include
./core/deps/llvm/build/lib
./core/deps/llvm/build/libexec
./core/deps/llvm/build/share
key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
restore-keys: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}

- name: Quit if cache miss
if: ${{ matrix.light == 'green' && steps.cache_llvm.outputs.cache-hit != 'true' }}
run: echo "::error::can not get prebuilt llvm libraries" && exit 1

- name: Extract the LLVM package
if: ${{ matrix.light == 'green' }}
run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
working-directory: ./core/deps/llvm/build

- name: Build wamrc
if: ${{ matrix.light == 'green' }}
run: |
Expand Down Expand Up @@ -293,19 +291,18 @@ jobs:
if: ${{ matrix.light == 'green' }}
uses: actions/cache@v2
with:
path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
path: |
./core/deps/llvm/build/bin
./core/deps/llvm/build/include
./core/deps/llvm/build/lib
./core/deps/llvm/build/libexec
./core/deps/llvm/build/share
key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
restore-keys: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}

- name: Quit if cache miss
if: ${{ matrix.light == 'green' && steps.cache_llvm.outputs.cache-hit != 'true' }}
run: echo "::error::can not get prebuilt llvm libraries" && exit 1

- name: Extract the LLVM package
if: ${{ matrix.light == 'green' }}
run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
working-directory: ./core/deps/llvm/build

- name: Build iwasm
if: ${{ matrix.light == 'green' }}
run: |
Expand Down Expand Up @@ -362,19 +359,18 @@ jobs:
if: ${{ matrix.light == 'green' }}
uses: actions/cache@v2
with:
path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
path: |
./core/deps/llvm/build/bin
./core/deps/llvm/build/include
./core/deps/llvm/build/lib
./core/deps/llvm/build/libexec
./core/deps/llvm/build/share
key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
restore-keys: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}

- name: Quit if cache miss
if: ${{ matrix.light == 'green' && steps.cache_llvm.outputs.cache-hit != 'true' }}
run: echo "::error::can not get prebuilt llvm libraries" && exit 1

- name: Extract the LLVM package
if: ${{ matrix.light == 'green' }}
run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
working-directory: ./core/deps/llvm/build

- name: download and install wabt
if: ${{ matrix.light == 'green' }}
run: |
Expand Down Expand Up @@ -440,19 +436,18 @@ jobs:
if: ${{ matrix.light == 'green' }}
uses: actions/cache@v2
with:
path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
path: |
./core/deps/llvm/build/bin
./core/deps/llvm/build/include
./core/deps/llvm/build/lib
./core/deps/llvm/build/libexec
./core/deps/llvm/build/share
key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
restore-keys: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}

- name: Quit if cache miss
if: ${{ matrix.light == 'green' && steps.cache_llvm.outputs.cache-hit != 'true' }}
run: echo "::error::can not get prebuilt llvm libraries" && exit 1

- name: Extract the LLVM package
if: ${{ matrix.light == 'green' }}
run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
working-directory: ./core/deps/llvm/build

- name: download and install wasi-sdk
if: ${{ matrix.light == 'green' }}
run: |
Expand Down
78 changes: 24 additions & 54 deletions .github/workflows/spec_test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,38 +49,12 @@ jobs:
with:
access_token: ${{ github.token }}

build_llvm_libraries:
needs: cancel_previous
runs-on: ubuntu-20.04
steps:
- name: checkout
uses: actions/checkout@v2

- name: Cache LLVM libraries
id: cache_llvm
uses: actions/cache@v2
with:
path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
restore-keys: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}

- name: Build llvm and clang from source
id: build_llvm
if: ${{ steps.cache_llvm.outputs.cache-hit != 'true' }}
run: /usr/bin/env python3 ./build_llvm.py --arch X86 WebAssembly
working-directory: build-scripts

- name: package LLVM
if: ${{ steps.build_llvm.conclusion == 'success' }}
run: mv LLVM-13.0.0-*.tar.gz LLVM-13.0.0.tar.gz
working-directory: core/deps/llvm/build/

spec_test_default:
needs: build_llvm_libraries
needs: cancel_previous
runs-on: ubuntu-20.04
strategy:
matrix:
test_option: [ $DEFAULT_TEST_OPTIONS, $SIMD_TEST_OPTIONS ]
test_option: [$DEFAULT_TEST_OPTIONS, $SIMD_TEST_OPTIONS]
steps:
- name: checkout
uses: actions/checkout@v2
Expand All @@ -89,17 +63,13 @@ jobs:
id: cache_llvm
uses: actions/cache@v2
with:
path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
path: |
./core/deps/llvm/build/bin
./core/deps/llvm/build/include
./core/deps/llvm/build/lib
./core/deps/llvm/build/libexec
./core/deps/llvm/build/share
key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
restore-keys: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}

- name: Quit if cache miss
if: steps.cache_llvm.outputs.cache-hit != 'true'
run: echo "::error::can not get prebuilt llvm libraries" && exit 1

- name: Extract the LLVM package
run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
working-directory: ./core/deps/llvm/build

- name: install Ninja
run: sudo apt install -y ninja-build
Expand All @@ -109,12 +79,12 @@ jobs:
working-directory: ./tests/wamr-test-suites

spec_test_extra:
needs: build_llvm_libraries
needs: cancel_previous
if: ${{ endsWith(github.repository, 'wasm-micro-runtime') }}
runs-on: ubuntu-20.04
strategy:
matrix:
test_option: [ $MULTI_MODULES_TEST_OPTIONS, $THREADS_TEST_OPTIONS ]
test_option: [$MULTI_MODULES_TEST_OPTIONS, $THREADS_TEST_OPTIONS]
steps:
- name: checkout
uses: actions/checkout@v2
Expand All @@ -123,18 +93,18 @@ jobs:
id: cache_llvm
uses: actions/cache@v2
with:
path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
path: |
./core/deps/llvm/build/bin
./core/deps/llvm/build/include
./core/deps/llvm/build/lib
./core/deps/llvm/build/libexec
./core/deps/llvm/build/share
key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
restore-keys: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}

- name: Quit if cache miss
if: steps.cache_llvm.outputs.cache-hit != 'true'
run: echo "::error::can not get prebuilt llvm libraries" && exit 1

- name: Extract the LLVM package
run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
working-directory: ./core/deps/llvm/build

- name: install Ninja
run: sudo apt install -y ninja-build

Expand All @@ -144,11 +114,11 @@ jobs:

spec_test_x86_32:
if: ${{ endsWith(github.repository, 'wasm-micro-runtime') }}
needs: build_llvm_libraries
needs: cancel_previous
runs-on: ubuntu-20.04
strategy:
matrix:
test_option: [ $DEFAULT_TEST_OPTIONS, $THREADS_TEST_OPTIONS ]
test_option: [$DEFAULT_TEST_OPTIONS, $THREADS_TEST_OPTIONS]
steps:
- name: checkout
uses: actions/checkout@v2
Expand All @@ -157,18 +127,18 @@ jobs:
id: cache_llvm
uses: actions/cache@v2
with:
path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
path: |
./core/deps/llvm/build/bin
./core/deps/llvm/build/include
./core/deps/llvm/build/lib
./core/deps/llvm/build/libexec
./core/deps/llvm/build/share
key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
restore-keys: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}

- name: Quit if cache miss
if: steps.cache_llvm.outputs.cache-hit != 'true'
run: echo "::error::can not get prebuilt llvm libraries" && exit 1

- name: Extract the LLVM package
run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
working-directory: ./core/deps/llvm/build

- name: install Ninja and x32 support libraries
run: sudo apt install -y g++-multilib libgcc-9-dev lib32gcc-9-dev ninja-build

Expand Down
48 changes: 39 additions & 9 deletions build-scripts/build_llvm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ def build_llvm(llvm_dir, platform, backends, projects):
lib_llvm_core_library = build_dir.joinpath("lib/libLLVMCore.a").resolve()
if lib_llvm_core_library.exists():
print(f"Please remove {build_dir} manually and try again")
return
return build_dir

compile_options = " ".join(
LLVM_COMPILE_OPTIONS
Expand All @@ -126,6 +126,28 @@ def build_llvm(llvm_dir, platform, backends, projects):
return build_dir


def repackage_llvm(llvm_dir):
build_dir = llvm_dir.joinpath("./build").resolve()

packs = [f for f in build_dir.glob("LLVM-13*.tar.gz")]
if len(packs) > 1:
raise Exception("Find more than one LLVM-13*.tar.gz")

if not packs:
return

llvm_package = packs[0].name
# mv build/LLVM-13.0.0*.gz .
shutil.move(str(build_dir.joinpath(llvm_package).resolve()), str(llvm_dir))
# rm -r build
shutil.rmtree(str(build_dir))
# mkdir build
build_dir.mkdir()
# tar xf ./LLVM-13.0.0-*.tar.gz --strip-components=1 --directory=build
CMD = f"tar xf {llvm_dir.joinpath(llvm_package).resolve()} --strip-components=1 --directory={build_dir}"
subprocess.check_call(shlex.split(CMD), cwd=llvm_dir)


def main():
parser = argparse.ArgumentParser(description="build necessary LLVM libraries")
parser.add_argument(
Expand Down Expand Up @@ -198,16 +220,24 @@ def main():
current_dir = current_file.parent.resolve()
deps_dir = current_dir.joinpath("../core/deps").resolve()

print(f"==================== CLONE LLVM ====================")
llvm_info = llvm_repo_and_branch.get(platform, llvm_repo_and_branch["default"])
llvm_dir = clone_llvm(deps_dir, llvm_info["repo"], llvm_info["branch"])
try:
print(f"==================== CLONE LLVM ====================")
llvm_info = llvm_repo_and_branch.get(platform, llvm_repo_and_branch["default"])
llvm_dir = clone_llvm(deps_dir, llvm_info["repo"], llvm_info["branch"])

print()
print(f"==================== BUILD LLVM ====================")
build_llvm(llvm_dir, platform, options.arch, options.project)

print()
print(f"==================== BUILD LLVM ====================")
build_llvm(llvm_dir, platform, options.arch, options.project)
print()
print(f"==================== PACKAGE LLVM ====================")
repackage_llvm(llvm_dir)

print()
print()
return True
except subprocess.CalledProcessError:
return False


if __name__ == "__main__":
main()
sys.exit(0 if main() else 1)