From 88ecab5a7799291796f7d372c77e5d2507eeed46 Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Wed, 27 Apr 2022 23:35:08 +0300 Subject: [PATCH] Github Actions: add linux-aarch64 release build --- .github/cross-linux-aarch64/Dockerfile | 12 +++ .github/workflows/build_linux_aarch64.yml | 75 ++++++++++++++++ .github/workflows/test-sys.yaml | 24 ++--- Makefile | 103 +++++++++++----------- 4 files changed, 148 insertions(+), 66 deletions(-) create mode 100644 .github/cross-linux-aarch64/Dockerfile create mode 100644 .github/workflows/build_linux_aarch64.yml diff --git a/.github/cross-linux-aarch64/Dockerfile b/.github/cross-linux-aarch64/Dockerfile new file mode 100644 index 00000000000..394652cbe2b --- /dev/null +++ b/.github/cross-linux-aarch64/Dockerfile @@ -0,0 +1,12 @@ +FROM rust:1 +#FROM ghcr.io/cross-rs/aarch64-unknown-linux-gnu:edge + +# set CROSS_DOCKER_IN_DOCKER to inform `cross` that it is executed from within a container +ENV CROSS_DOCKER_IN_DOCKER=true + +RUN cargo install cross +RUN dpkg --add-architecture arm64 && \ + apt-get update && \ + apt-get install -qy curl && \ + curl -sSL https://get.docker.com/ | sh && \ + apt-get install --assume-yes libxkbcommon0:arm64 libwayland-cursor0:arm64 libxkbcommon-dev:arm64 libwayland-dev:arm64 libxkbcommon-x11-dev:arm64 diff --git a/.github/workflows/build_linux_aarch64.yml b/.github/workflows/build_linux_aarch64.yml new file mode 100644 index 00000000000..2623c754430 --- /dev/null +++ b/.github/workflows/build_linux_aarch64.yml @@ -0,0 +1,75 @@ +on: [push] + +name: linux-aarch64 build + +jobs: + linux_aarch64: + name: Linux aarch64 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + target: aarch64-unknown-linux-gnu + override: true + - name: Build cross image + run: | + #cargo install cross + docker build -t wasmer/aarch64 /home/runner/work/wasmer/wasmer/.github/cross-linux-aarch64/ + #cat << EOF > ./Cross.toml + #[target.aarch64-unknown-linux-gnu] + #image = "wasmer/aarch64" + #EOF + env: + CROSS_DOCKER_IN_DOCKER: true + - name: Install LLVM + shell: bash + run: | + sudo apt-get update -y + sudo apt-get install -yq tree + curl --proto '=https' --tlsv1.2 -sSf "https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.0/clang+llvm-13.0.0-aarch64-linux-gnu.tar.xz" -L -o llvm.tar.xz + LLVM_DIR=$(pwd)/${{ env.LLVM_DIR }} + mkdir ${LLVM_DIR} + tar xf llvm.tar.xz --strip-components=1 -C ${LLVM_DIR} + echo "${LLVM_DIR}/bin" >> $GITHUB_PATH + echo "LLVM_SYS_120_PREFIX=${LLVM_DIR}" >> $GITHUB_ENV + env: + LLVM_DIR: .llvm + - name: Build Wasmer binary + run: | + make build-wasmer + env: + CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v /home/runner/work/wasmer/wasmer:/project -w /project wasmer/aarch64 cross + CROSS_DOCKER_IN_DOCKER: true + CARGO_TARGET: --target aarch64-unknown-linux-gnu + PKG_CONFIG_PATH: /usr/lib/aarch64-linux-gnu/pkgconfig + PKG_CONFIG_ALLOW_CROSS: true + - name: Build C API + run: | + make build-capi + env: + CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v /home/runner/work/wasmer/wasmer:/project -w /project wasmer/aarch64 cross + CROSS_DOCKER_IN_DOCKER: true + CARGO_TARGET: --target aarch64-unknown-linux-gnu + PKG_CONFIG_PATH: /usr/lib/aarch64-linux-gnu/pkgconfig + PKG_CONFIG_ALLOW_CROSS: true + - name: Dist + run: | + #tree target/ + make distribution + env: + CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v /home/runner/work/wasmer/wasmer:/project -w /project wasmer/aarch64 cross + CROSS_DOCKER_IN_DOCKER: true + CARGO_TARGET: --target aarch64-unknown-linux-gnu + PKG_CONFIG_PATH: /usr/lib/aarch64-linux-gnu/pkgconfig + PKG_CONFIG_ALLOW_CROSS: true + TARGET: aarch64-unknown-linux-gnu + TARGET_DIR: target/aarch64-unknown-linux-gnu/release + - name: Upload Artifacts + uses: actions/upload-artifact@v2 + with: + name: wasmer.tar.gz + path: dist + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/test-sys.yaml b/.github/workflows/test-sys.yaml index 84686ad4ac7..db68cd29e2c 100644 --- a/.github/workflows/test-sys.yaml +++ b/.github/workflows/test-sys.yaml @@ -76,13 +76,6 @@ jobs: use_sccache: true run_test: true run_test_capi: false # We can't run yet the capi tests on Windows - #- build: linux-aarch64 - # os: [self-hosted, linux, ARM64] - # random_sccache_port: true - # use_sccache: true - # llvm_url: 'https://github.com/wasmerio/llvm-custom-builds/releases/download/12.x/linux-aarch64.tar.gz' - # artifact_name: 'wasmer-linux-aarch64' - # run_integration_tests: false - build: linux-musl-x64 os: ubuntu-latest container: alpine:latest @@ -247,8 +240,16 @@ jobs: echo "rpath = false" >> Cargo.toml make build-wasmer-headless-minimal - name: Dist + if: matrix.build != 'macos-arm64' run: | make distribution + - name: Dist macos-arm64 + if: matrix.build == 'macos-arm64' + run: | + make distribution + env: + TARGET: aarch64-apple-darwin + TARGET_DIR: target/aarch64-apple-darwin/release - name: Run integration tests (Windows) shell: cmd run: | @@ -438,15 +439,6 @@ jobs: asset_path: artifacts/wasmer-darwin-arm64/wasmer.tar.gz asset_name: wasmer-darwin-arm64.tar.gz asset_content_type: application/gzip - #- name: Upload Release Asset Linux aarch64 - # uses: actions/upload-release-asset@v1 - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # with: - # upload_url: ${{ steps.create_release.outputs.upload_url }} - # asset_path: artifacts/wasmer-linux-aarch64/wasmer.tar.gz - # asset_name: wasmer-linux-aarch64.tar.gz - # asset_content_type: application/gzip audit: name: Audit diff --git a/Makefile b/Makefile index 00b2a1c1c1d..37bcfe40433 100644 --- a/Makefile +++ b/Makefile @@ -98,12 +98,15 @@ endif # ##### +CARGO_BINARY ?= cargo +CARGO_TARGET ?= # Variables that can be overriden by the users to force to enable or # to disable a specific compiler. ENABLE_CRANELIFT ?= ENABLE_LLVM ?= ENABLE_SINGLEPASS ?= +LLVM_VERSION ?= # Which compilers we build. These have dependencies that may not be on the system. compilers := @@ -127,11 +130,11 @@ endif ifneq ($(ENABLE_LLVM), 0) # … then maybe the user forced to enable the LLVM compiler. ifeq ($(ENABLE_LLVM), 1) - LLVM_VERSION := $(shell llvm-config --version) + LLVM_VERSION ?= $(shell llvm-config --version) compilers += llvm # … otherwise, we try to autodetect LLVM from `llvm-config` else ifneq (, $(shell which llvm-config 2>/dev/null)) - LLVM_VERSION := $(shell llvm-config --version) + LLVM_VERSION ?= $(shell llvm-config --version) # If findstring is not empty, then it have found the value ifneq (, $(findstring 13,$(LLVM_VERSION))) @@ -142,10 +145,10 @@ ifneq ($(ENABLE_LLVM), 0) # … or try to autodetect LLVM from `llvm-config-`. else ifneq (, $(shell which llvm-config-13 2>/dev/null)) - LLVM_VERSION := $(shell llvm-config-13 --version) + LLVM_VERSION ?= $(shell llvm-config-13 --version) compilers += llvm else ifneq (, $(shell which llvm-config-12 2>/dev/null)) - LLVM_VERSION := $(shell llvm-config-12 --version) + LLVM_VERSION ?= $(shell llvm-config-12 --version) compilers += llvm endif endif @@ -304,10 +307,10 @@ endif HOST_TARGET=$(shell rustup show | grep 'Default host: ' | cut -d':' -f2 | tr -d ' ') -TARGET_DIR := target/release +TARGET_DIR ?= target/release ifneq (, $(TARGET)) - TARGET_DIR := target/$(TARGET)/release + TARGET_DIR ?= target/$(TARGET)/release endif $(info -----------) @@ -371,13 +374,13 @@ endif all: build-wasmer build-capi build-wasmer: - cargo build --release --manifest-path lib/cli/Cargo.toml $(compiler_features) --bin wasmer + $(CARGO_BINARY) build $(CARGO_TARGET) --release --manifest-path lib/cli/Cargo.toml $(compiler_features) --bin wasmer build-wasmer-debug: - cargo build --manifest-path lib/cli/Cargo.toml $(compiler_features) --features "debug" --bin wasmer + $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/cli/Cargo.toml $(compiler_features) --features "debug" --bin wasmer bench: - cargo bench $(compiler_features) + $(CARGO_BINARY) bench $(CARGO_TARGET) $(compiler_features) # For best results ensure the release profile looks like the following # in Cargo.toml: @@ -409,13 +412,13 @@ get-wapm: build-wapm: get-wapm ifeq ($(IS_DARWIN), 1) # We build it without bundling sqlite, as is included by default in macos - cargo build --release --manifest-path wapm-cli/Cargo.toml --no-default-features --features "full packagesigning telemetry update-notifications" + $(CARGO_BINARY) build $(CARGO_TARGET) --release --manifest-path wapm-cli/Cargo.toml --no-default-features --features "full packagesigning telemetry update-notifications" else - cargo build --release --manifest-path wapm-cli/Cargo.toml --features "telemetry update-notifications" + $(CARGO_BINARY) build $(CARGO_TARGET) --release --manifest-path wapm-cli/Cargo.toml --features "telemetry update-notifications" endif build-docs: - cargo doc --release $(compiler_features) --document-private-items --no-deps --workspace --exclude wasmer-c-api + $(CARGO_BINARY) doc $(CARGO_TARGET) --release $(compiler_features) --document-private-items --no-deps --workspace --exclude wasmer-c-api capi-setup: ifeq ($(IS_WINDOWS), 1) @@ -427,77 +430,77 @@ build-docs-capi: capi-setup # when generating the documentation, we rename it to its # crate's name. Then we restore the lib's name. sed "$(SEDI)" -e 's/name = "wasmer" # ##lib.name##/name = "wasmer_c_api" # ##lib.name##/' lib/c-api/Cargo.toml - RUSTFLAGS="${RUSTFLAGS}" cargo doc --manifest-path lib/c-api/Cargo.toml --no-deps --features wat,universal,staticlib,dylib,cranelift,wasi + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) doc $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --no-deps --features wat,universal,staticlib,dylib,cranelift,wasi sed "$(SEDI)" -e 's/name = "wasmer_c_api" # ##lib.name##/name = "wasmer" # ##lib.name##/' lib/c-api/Cargo.toml build-capi: capi-setup - RUSTFLAGS="${RUSTFLAGS}" cargo build --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features wat,universal,dylib,staticlib,wasi,middlewares $(capi_compiler_features) build-capi-singlepass: capi-setup - RUSTFLAGS="${RUSTFLAGS}" cargo build --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features wat,universal,dylib,staticlib,singlepass,wasi,middlewares build-capi-singlepass-universal: capi-setup - RUSTFLAGS="${RUSTFLAGS}" cargo build --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features wat,universal,singlepass,wasi,middlewares build-capi-singlepass-dylib: capi-setup - RUSTFLAGS="${RUSTFLAGS}" cargo build --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features wat,dylib,singlepass,wasi,middlewares build-capi-singlepass-staticlib: capi-setup - RUSTFLAGS="${RUSTFLAGS}" cargo build --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features wat,staticlib,singlepass,wasi,middlewares build-capi-cranelift: capi-setup - RUSTFLAGS="${RUSTFLAGS}" cargo build --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features wat,universal,dylib,staticlib,cranelift,wasi,middlewares build-capi-cranelift-universal: capi-setup - RUSTFLAGS="${RUSTFLAGS}" cargo build --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features wat,universal,cranelift,wasi,middlewares build-capi-cranelift-dylib: capi-setup - RUSTFLAGS="${RUSTFLAGS}" cargo build --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features wat,dylib,cranelift,wasi,middlewares build-capi-cranelift-staticlib: capi-setup - RUSTFLAGS="${RUSTFLAGS}" cargo build --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features wat,dylib,staticlib,cranelift,wasi,middlewares build-capi-llvm: capi-setup - RUSTFLAGS="${RUSTFLAGS}" cargo build --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features wat,universal,dylib,staticlib,llvm,wasi,middlewares build-capi-llvm-universal: capi-setup - RUSTFLAGS="${RUSTFLAGS}" cargo build --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features wat,universal,llvm,wasi,middlewares build-capi-llvm-dylib: capi-setup - RUSTFLAGS="${RUSTFLAGS}" cargo build --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features wat,dylib,llvm,wasi,middlewares build-capi-llvm-staticlib: capi-setup - RUSTFLAGS="${RUSTFLAGS}" cargo build --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features wat,staticlib,llvm,wasi,middlewares # Headless (we include the minimal to be able to run) build-capi-headless-universal: capi-setup - RUSTFLAGS="${RUSTFLAGS}" cargo build --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features universal,wasi build-capi-headless-dylib: capi-setup - RUSTFLAGS="${RUSTFLAGS}" cargo build --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features dylib,wasi build-capi-headless-staticlib: capi-setup - RUSTFLAGS="${RUSTFLAGS}" cargo build --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features staticlib,wasi build-capi-headless-all: capi-setup - RUSTFLAGS="${RUSTFLAGS}" cargo build --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features universal,dylib,staticlib,wasi build-capi-headless-ios: capi-setup @@ -513,13 +516,13 @@ build-capi-headless-ios: capi-setup test: test-compilers test-packages test-examples test-compilers: - cargo test --release --tests $(compiler_features) + $(CARGO_BINARY) test $(CARGO_TARGET) --release --tests $(compiler_features) test-packages: - cargo test --all --release $(exclude_tests) - cargo test --manifest-path lib/compiler-cranelift/Cargo.toml --release --no-default-features --features=std - cargo test --manifest-path lib/compiler-singlepass/Cargo.toml --release --no-default-features --features=std - cargo test --manifest-path lib/cli/Cargo.toml $(compiler_features) --release + $(CARGO_BINARY) test $(CARGO_TARGET) --all --release $(exclude_tests) + $(CARGO_BINARY) test $(CARGO_TARGET) --manifest-path lib/compiler-cranelift/Cargo.toml --release --no-default-features --features=std + $(CARGO_BINARY) test $(CARGO_TARGET) --manifest-path lib/compiler-singlepass/Cargo.toml --release --no-default-features --features=std + $(CARGO_BINARY) test $(CARGO_TARGET) --manifest-path lib/cli/Cargo.toml $(compiler_features) --release test-js: test-js-api test-js-wasi @@ -538,22 +541,22 @@ test-js-wasi: test-compilers-compat: $(foreach compiler,$(compilers),test-$(compiler)) test-singlepass-dylib: - cargo test --release --tests $(compiler_features) -- singlepass::dylib + $(CARGO_BINARY) test $(CARGO_TARGET) --release --tests $(compiler_features) -- singlepass::dylib test-singlepass-universal: - cargo test --release --tests $(compiler_features) -- singlepass::universal + $(CARGO_BINARY) test $(CARGO_TARGET) --release --tests $(compiler_features) -- singlepass::universal test-cranelift-dylib: - cargo test --release --tests $(compiler_features) -- cranelift::dylib + $(CARGO_BINARY) test $(CARGO_TARGET) --release --tests $(compiler_features) -- cranelift::dylib test-cranelift-universal: - cargo test --release --tests $(compiler_features) -- cranelift::universal + $(CARGO_BINARY) test $(CARGO_TARGET) --release --tests $(compiler_features) -- cranelift::universal test-llvm-dylib: - cargo test --release --tests $(compiler_features) -- llvm::dylib + $(CARGO_BINARY) test $(CARGO_TARGET) --release --tests $(compiler_features) -- llvm::dylib test-llvm-universal: - cargo test --release --tests $(compiler_features) -- llvm::universal + $(CARGO_BINARY) test $(CARGO_TARGET) --release --tests $(compiler_features) -- llvm::universal test-singlepass: $(foreach singlepass_engine,$(filter singlepass-%,$(compilers_engines)),test-$(singlepass_engine)) @@ -566,7 +569,7 @@ test-llvm: $(foreach llvm_engine,$(filter llvm-%,$(compilers_engines)),test-$(ll test-capi: build-capi package-capi $(foreach compiler_engine,$(capi_compilers_engines),test-capi-crate-$(compiler_engine) test-capi-integration-$(compiler_engine)) test-capi-crate-%: - WASMER_CAPI_CONFIG=$(shell echo $@ | sed -e s/test-capi-crate-//) cargo test --manifest-path lib/c-api/Cargo.toml --release \ + WASMER_CAPI_CONFIG=$(shell echo $@ | sed -e s/test-capi-crate-//) $(CARGO_BINARY) test $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features wat,universal,dylib,staticlib,wasi,middlewares $(capi_compiler_features) -- --nocapture test-capi-integration-%: @@ -576,25 +579,25 @@ test-capi-integration-%: cd lib/c-api/examples; WASMER_CAPI_CONFIG=$(shell echo $@ | sed -e s/test-capi-integration-//) WASMER_DIR=`pwd`/../../../package make run test-wasi-unit: - cargo test --manifest-path lib/wasi/Cargo.toml --release + $(CARGO_BINARY) test $(CARGO_TARGET) --manifest-path lib/wasi/Cargo.toml --release test-wasi: - cargo test --release --tests $(compiler_features) -- wasi::wasitests + $(CARGO_BINARY) test $(CARGO_TARGET) --release --tests $(compiler_features) -- wasi::wasitests test-examples: - cargo test $(compiler_features) --features wasi --examples - cargo test --release $(compiler_features) --features wasi --examples + $(CARGO_BINARY) test $(CARGO_TARGET) $(compiler_features) --features wasi --examples + $(CARGO_BINARY) test $(CARGO_TARGET) --release $(compiler_features) --features wasi --examples test-integration: - cargo test -p wasmer-integration-tests-cli + $(CARGO_BINARY) test $(CARGO_TARGET) -p wasmer-integration-tests-cli test-integration-ios: - cargo test -p wasmer-integration-tests-ios + $(CARGO_BINARY) test $(CARGO_TARGET) -p wasmer-integration-tests-ios generate-wasi-tests: # Uncomment the following for installing the toolchain # cargo run -p wasi-test-generator -- -s - cargo run -p wasi-test-generator -- -g + $(CARGO_BINARY) run $(CARGO_TARGET) -p wasi-test-generator -- -g ##### # # Packaging. @@ -672,7 +675,7 @@ package-docs: build-docs build-docs-capi echo '' > package/docs/index.html echo '' > package/docs/crates/index.html -package: package-wapm package-wasmer package-minimal-headless-wasmer package-capi +package: package-wasmer package-minimal-headless-wasmer package-capi distribution: package cp LICENSE package/LICENSE