From 52901850a9c6cc5b5a8e7c70d396302879a9b46f Mon Sep 17 00:00:00 2001 From: jubianchi Date: Wed, 6 Jan 2021 21:21:03 +0100 Subject: [PATCH] chore: Build Wasmer on musl Closes #1482 Closes #1766 --- .github/workflows/main.yaml | 100 +++++++++++++++++++++++++++--------- Makefile | 67 ++++++++++++++---------- bors.toml | 1 + lib/c-api/build.rs | 17 +++++- lib/c-api/tests/.gitignore | 5 ++ 5 files changed, 137 insertions(+), 53 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 10fda7db182..dfe46b1bdb1 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -38,7 +38,6 @@ jobs: strategy: fail-fast: false matrix: - build: [linux-x64, macos-x64, macos-arm64, windows-x64, linux-aarch64] include: - build: linux-x64 os: ubuntu-18.04 @@ -77,14 +76,15 @@ jobs: CARGO_SCCACHE_VERSION: 0.2.14-alpha.0-parity SCCACHE_AZURE_BLOB_CONTAINER: wasmerstoragesccacheblob SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} + TARGET: ${{ matrix.target }} steps: - uses: actions/checkout@v2 - name: Set up libstdc++ on Linux + if: matrix.build == 'linux-x64' run: | sudo apt-get update -y sudo apt-get install -y --allow-downgrades libstdc++6=8.4.0-1ubuntu1~18.04 sudo apt-get install --reinstall g++-8 - if: matrix.os == 'ubuntu-18.04' - name: Install Rust ${{ matrix.rust }} uses: actions-rs/toolchain@v1 with: @@ -125,7 +125,6 @@ jobs: ~/.cargo/registry ~/.cargo/git key: ${{ matrix.build }}-${{ matrix.target }}-cargo-${{ hashFiles('Cargo.lock') }}-v1 - # Install sccache - uses: actions/cache@v2 with: path: ${{ runner.tool_cache }}/cargo-sccache @@ -173,7 +172,7 @@ jobs: - name: Test run: | make test - if: matrix.target != 'aarch64-apple-darwin' + if: matrix.build != 'macos-arm64' - name: Test C API run: | make test-capi @@ -190,9 +189,9 @@ jobs: run: | make build-wasmer - name: Build Wapm binary + if: needs.setup.outputs.DOING_RELEASE == '1' run: | make build-wapm - if: needs.setup.outputs.DOING_RELEASE == '1' - name: Install Nightly Rust for Headless uses: actions-rs/toolchain@v1 with: @@ -202,6 +201,7 @@ jobs: components: "rust-src" if: needs.setup.outputs.DOING_RELEASE == '1' - name: Build Minimal Wasmer Headless + if: needs.setup.outputs.DOING_RELEASE == '1' run: | cargo install xargo echo "" >> Cargo.toml @@ -216,17 +216,6 @@ jobs: echo "codegen-units = 1" >> Cargo.toml echo "rpath = false" >> Cargo.toml make build-wasmer-headless-minimal - if: needs.setup.outputs.DOING_RELEASE == '1' - - name: Copy target binaries - run: | - mkdir -p target/release - cp target/${{matrix.target}}/release/wasmer* target/release - cp target/${{matrix.target}}/release/libwasmer* target/release - if [ -d "wapm-cli" ]; then - mkdir -p wapm-cli/target/release - cp wapm-cli/target/${{matrix.target}}/release/wapm* wapm-cli/target/release - fi - if: matrix.target - name: Dist run: | make distribution @@ -243,7 +232,7 @@ jobs: make test-integration if: matrix.run_integration_tests && matrix.os != 'windows-latest' - name: Cross compile from Linux - if: matrix.os == 'ubuntu-18.04' + if: matrix.build == 'linux-x64' shell: bash run: | ls target/release @@ -290,6 +279,64 @@ jobs: if-no-files-found: error retention-days: 1 + test-docker: + name: Test on ${{ matrix.build }} + runs-on: ubuntu-latest + needs: setup + strategy: + fail-fast: false + matrix: + include: + - build: linux-musl-x64 + image: alpine:latest + rust: 1.49 + artifact_name: 'wasmer-linux-musl-amd64' + steps: + - uses: actions/checkout@v2 + + - uses: addnab/docker-run-action@v1 + with: + image: ${{ matrix.image }} + options: -v ${{ github.workspace }}:/work + run: | + set -e + + # Set up tools + apk add musl-dev curl make libtool libffi-dev gcc automake autoconf git + + # Install Rust ${{ matrix.rust }} + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup.sh + sh ./rustup.sh -y + source ~/.cargo/env + rm -f ./rustup.sh + + # Set up Rust + rustup toolchain install ${{ matrix.rust }} + rustup default ${{ matrix.rust }} + + # Change working directory + cd /work + + # Test + make test test-capi + + # Build C API, Wasmer & Wapm + make build-capi build-wasmer build-wapm + + # Dist + make distribution + + # Run integration tests + export WASMER_DIR=`pwd`/package + make test-integration + - name: Upload Artifacts + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.artifact_name }} + path: dist + if-no-files-found: error + retention-days: 1 + test-cross-compile-on-linux: name: Test cross-compile on linux needs: [setup, test] @@ -340,7 +387,7 @@ jobs: # wasmer run artifacts/cross/cross_compiled_from_mac/win_from_mac.wjit -- --eval "function greet(name) { return JSON.stringify('Hello, ' + name); }; print(greet('World'));" release: - needs: [setup, test, test-cross-compile-on-linux, test-cross-compile-on-mac] #, test-cross-compile-on-win] + needs: [setup, test, test-docker, test-cross-compile-on-linux, test-cross-compile-on-mac] #, test-cross-compile-on-win] runs-on: ubuntu-latest if: needs.setup.outputs.DOING_RELEASE == '1' steps: @@ -348,7 +395,6 @@ jobs: uses: actions/download-artifact@v2 with: path: artifacts - - name: Create Release id: create_release uses: actions/create-release@v1 @@ -359,7 +405,6 @@ jobs: release_name: Release ${{ needs.setup.outputs.VERSION }} draft: true prerelease: false - - name: Upload Release Asset Windows Installer uses: actions/upload-release-asset@v1 env: @@ -369,7 +414,6 @@ jobs: asset_path: artifacts/wasmer-windows-amd64/WasmerInstaller.exe asset_name: wasmer-windows.exe asset_content_type: application/vnd.microsoft.portable-executable - - name: Upload Release Asset Windows uses: actions/upload-release-asset@v1 env: @@ -379,7 +423,6 @@ jobs: asset_path: artifacts/wasmer-windows-amd64/wasmer.tar.gz asset_name: wasmer-windows-amd64.tar.gz asset_content_type: application/gzip - - name: Upload Release Asset Linux amd64 uses: actions/upload-release-asset@v1 env: @@ -389,7 +432,16 @@ jobs: asset_path: artifacts/wasmer-linux-amd64/wasmer.tar.gz asset_name: wasmer-linux-amd64.tar.gz asset_content_type: application/gzip - + - name: Upload Release Asset Linux amd64 (musl) + id: upload-release-asset-linux-musl-amd64 + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: artifacts/wasmer-linux-musl-amd64/wasmer.tar.gz + asset_name: wasmer-linux-musl-amd64.tar.gz + asset_content_type: application/gzip - name: Upload Release Asset Mac amd64 uses: actions/upload-release-asset@v1 env: @@ -399,7 +451,6 @@ jobs: asset_path: artifacts/wasmer-darwin-amd64/wasmer.tar.gz asset_name: wasmer-darwin-amd64.tar.gz asset_content_type: application/gzip - - name: Upload Release Asset Mac arm64 uses: actions/upload-release-asset@v1 env: @@ -409,7 +460,6 @@ jobs: asset_path: artifacts/wasmer-darwin-arm64/wasmer.tar.gz asset_name: wasmer-darwin-arm64.tar.gz asset_content_type: application/gzip - - name: Upload Release Asset Linux aarch64 uses: actions/upload-release-asset@v1 env: diff --git a/Makefile b/Makefile index e451ece3b50..d539f69d2de 100644 --- a/Makefile +++ b/Makefile @@ -295,21 +295,34 @@ capi_compiler_features := --features $(subst $(space),$(comma),$(filter-out llvm ifneq (, $(filter 1, $(IS_DARWIN) $(IS_LINUX))) bold := $(shell tput bold 2>/dev/null || echo -n '') green := $(shell tput setaf 2 2>/dev/null || echo -n '') + yellow := $(shell tput setaf 3 2>/dev/null || echo -n '') reset := $(shell tput sgr0 2>/dev/null || echo -n '') endif HOST_TARGET=$(shell rustup show | grep 'Default host: ' | cut -d':' -f2 | tr -d ' ') +TARGET_DIR := target/release + +ifneq (, $(TARGET)) + TARGET_DIR := target/$(TARGET)/release +endif + $(info -----------) $(info $(bold)$(green)INFORMATION$(reset)) $(info -----------) $(info ) $(info Host Target: `$(bold)$(green)$(HOST_TARGET)$(reset)`.) +ifneq (, $(TARGET)) + # We use spaces instead of tabs to indent `$(info)` + # otherwise it's considered as a command outside a + # target and it will fail. + $(info Build Target: $(bold)$(green)$(TARGET)$(reset) $(yellow)($(TARGET_DIR))$(reset)) +endif ifneq (, $(LIBC)) - # We use spaces instead of tabs to indent `$(info)` - # otherwise it's considered as a command outside a - # target and it will fail. - $(info C standard library: $(bold)$(green)$(LIBC)$(reset)) + # We use spaces instead of tabs to indent `$(info)` + # otherwise it's considered as a command outside a + # target and it will fail. + $(info C standard library: $(bold)$(green)$(LIBC)$(reset)) endif $(info Enabled Compilers: $(bold)$(green)$(subst $(space),$(reset)$(comma)$(space)$(bold)$(green),$(compilers))$(reset).) $(info Compilers + engines pairs (for testing): $(bold)$(green)${compilers_engines}$(reset)) @@ -325,7 +338,6 @@ $(info --------------) $(info ) $(info ) - ############ # Building # ############ @@ -579,13 +591,13 @@ package-wapm: mkdir -p "package/bin" ifneq (, $(filter 1, $(IS_DARWIN) $(IS_LINUX))) if [ -d "wapm-cli" ]; then \ - cp wapm-cli/target/release/wapm package/bin/ ;\ + cp wapm-cli/$(TARGET_DIR)/wapm package/bin/ ;\ echo "#!/bin/bash\nwapm execute \"\$$@\"" > package/bin/wax ;\ chmod +x package/bin/wax ;\ fi else if [ -d "wapm-cli" ]; then \ - cp wapm-cli/target/release/wapm package/bin/ ;\ + cp wapm-cli/$(TARGET_DIR)/wapm package/bin/ ;\ fi ifeq ($(IS_DARWIN), 1) codesign -s - package/bin/wapm @@ -606,9 +618,9 @@ endif package-wasmer: mkdir -p "package/bin" ifeq ($(IS_WINDOWS), 1) - cp target/release/wasmer.exe package/bin/ + cp $(TARGET_DIR)/wasmer.exe package/bin/ else - cp target/release/wasmer package/bin/ + cp $(TARGET_DIR)/wasmer package/bin/ ifeq ($(IS_DARWIN), 1) codesign -s - package/bin/wasmer endif @@ -621,25 +633,28 @@ package-capi: cp lib/c-api/wasmer_wasm.h* package/include cp lib/c-api/wasm.h* package/include cp lib/c-api/README.md package/include/README.md -ifeq ($(IS_WINDOWS), 1) - cp target/release/wasmer_c_api.dll package/lib/wasmer.dll - cp target/release/wasmer_c_api.lib package/lib/wasmer.lib -else -ifeq ($(IS_DARWIN), 1) + + # Windows + if [ -f $(TARGET_DIR)/wasmer_c_api.dll ]; then \ + cp $(TARGET_DIR)/wasmer_c_api.dll package/lib/wasmer.dll ;\ + fi + if [ -f $(TARGET_DIR)/wasmer_c_api.lib ]; then \ + cp $(TARGET_DIR)/wasmer_c_api.lib package/lib/wasmer.lib ;\ + fi + # For some reason in macOS arm64 there are issues if we copy constantly in the install_name_tool util rm -f package/lib/libwasmer.dylib - cp target/release/libwasmer_c_api.dylib package/lib/libwasmer.dylib - cp target/release/libwasmer_c_api.a package/lib/libwasmer.a - # Fix the rpath for the dylib - install_name_tool -id "@rpath/libwasmer.dylib" package/lib/libwasmer.dylib -else - # In some cases the .so may not be available, for example when building against musl (static linking) - if [ -f target/release/libwasmer_c_api.so ]; then \ - cp target/release/libwasmer_c_api.so package/lib/libwasmer.so ;\ - fi; - cp target/release/libwasmer_c_api.a package/lib/libwasmer.a -endif -endif + if [ -f $(TARGET_DIR)/libwasmer_c_api.dylib ]; then \ + cp $(TARGET_DIR)/libwasmer_c_api.dylib package/lib/libwasmer.dylib ;\ + install_name_tool -id "@rpath/libwasmer.dylib" package/lib/libwasmer.dylib ;\ + fi + + if [ -f $(TARGET_DIR)/libwasmer_c_api.so ]; then \ + cp $(TARGET_DIR)/libwasmer_c_api.so package/lib/libwasmer.so ;\ + fi + if [ -f $(TARGET_DIR)/libwasmer_c_api.a ]; then \ + cp $(TARGET_DIR)/libwasmer_c_api.a package/lib/libwasmer.a ;\ + fi package-docs: build-docs build-docs-capi mkdir -p "package/docs" diff --git a/bors.toml b/bors.toml index c84a3a47f74..71ed41f6699 100644 --- a/bors.toml +++ b/bors.toml @@ -2,6 +2,7 @@ status = [ "Audit", "Code lint", "Test on linux-x64", + "Test on linux-musl-x64", "Test on linux-aarch64", "Test on macos-x64", "Test on windows-x64", diff --git a/lib/c-api/build.rs b/lib/c-api/build.rs index f5abe35d14f..43b348339a1 100644 --- a/lib/c-api/build.rs +++ b/lib/c-api/build.rs @@ -4,7 +4,10 @@ //! * setting `inline-c` up. use cbindgen::{Builder, Language}; -use std::{env, fs, path::PathBuf}; +use std::{ + env, fs, + path::{Path, PathBuf}, +}; const PRE_HEADER: &'static str = r#" // Define the `ARCH_X86_X64` constant. @@ -569,7 +572,17 @@ fn build_inline_c_env_vars() { } else if cfg!(target_os = "macos") { "libwasmer_c_api.dylib".to_string() } else { - "libwasmer_c_api.so".to_string() + let path = format!( + "{shared_object_dir}/{lib}", + shared_object_dir = shared_object_dir, + lib = "libwasmer_c_api.so" + ); + + if Path::new(path.as_str()).exists() { + "libwasmer_c_api.so".to_string() + } else { + "libwasmer_c_api.a".to_string() + } } ); } diff --git a/lib/c-api/tests/.gitignore b/lib/c-api/tests/.gitignore index 36e000ce2e2..58fbd3c5e8d 100644 --- a/lib/c-api/tests/.gitignore +++ b/lib/c-api/tests/.gitignore @@ -1,6 +1,11 @@ # ignore wasm-c-api binaries wasm-c-api-* test-* +wasm-c-api/example/* # Unignore files ending with `.c` (i.e. `wasm-c-api-wasi.c`) !*.c +!wasm-c-api/example/*.c +!wasm-c-api/example/*.cc +!wasm-c-api/example/*.wasm +!wasm-c-api/example/*.wat