Merge #2003

2003: chore: Build Wasmer on musl r=jubianchi a=jubianchi This patch adds a specific build for musl. Currently, it will only support JIT engine. I also changes the workflow definition a bit so we don't depend on the OS name but rather on the environment ID which is our convention. Closes #1482 Closes #1766  # Description  # Review - [ ] Add a short description of the the change to the CHANGELOG.md file Co-authored-by: jubianchi <[email protected]>
wasmerio · Mar 5, 2021 · ea4ea47 · ea4ea47
2 parents 1db36fb + 5290185
commit ea4ea47
Show file tree

Hide file tree

Showing 5 changed files with 137 additions and 53 deletions.
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -38,7 +38,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        build: [linux-x64, macos-x64, macos-arm64, windows-x64, linux-aarch64]
         include:
           - build: linux-x64
             os: ubuntu-18.04
@@ -77,14 +76,15 @@ jobs:
       CARGO_SCCACHE_VERSION: 0.2.14-alpha.0-parity
       SCCACHE_AZURE_BLOB_CONTAINER: wasmerstoragesccacheblob
       SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }}
+      TARGET: ${{ matrix.target }}
     steps:
       - uses: actions/checkout@v2
       - name: Set up libstdc++ on Linux
+        if: matrix.build == 'linux-x64'
         run: |
           sudo apt-get update -y
           sudo apt-get install -y --allow-downgrades libstdc++6=8.4.0-1ubuntu1~18.04
           sudo apt-get install --reinstall g++-8
-        if: matrix.os == 'ubuntu-18.04'
       - name: Install Rust ${{ matrix.rust }}
         uses: actions-rs/toolchain@v1
         with:
@@ -125,7 +125,6 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
           key: ${{ matrix.build }}-${{ matrix.target }}-cargo-${{ hashFiles('Cargo.lock') }}-v1
-      # Install sccache
       - uses: actions/cache@v2
         with:
           path: ${{ runner.tool_cache }}/cargo-sccache
@@ -173,7 +172,7 @@ jobs:
       - name: Test
         run: |
           make test
-        if: matrix.target != 'aarch64-apple-darwin'
+        if: matrix.build != 'macos-arm64'
       - name: Test C API
         run: |
           make test-capi
@@ -190,9 +189,9 @@ jobs:
         run: |
           make build-wasmer
       - name: Build Wapm binary
+        if: needs.setup.outputs.DOING_RELEASE == '1'
         run: |
           make build-wapm
-        if: needs.setup.outputs.DOING_RELEASE == '1'
       - name: Install Nightly Rust for Headless
         uses: actions-rs/toolchain@v1
         with:
@@ -202,6 +201,7 @@ jobs:
           components: "rust-src"
         if: needs.setup.outputs.DOING_RELEASE == '1'
       - name: Build Minimal Wasmer Headless
+        if: needs.setup.outputs.DOING_RELEASE == '1'
         run: |
           cargo install xargo
           echo "" >> Cargo.toml
@@ -216,17 +216,6 @@ jobs:
           echo "codegen-units = 1" >> Cargo.toml
           echo "rpath = false" >> Cargo.toml
           make build-wasmer-headless-minimal
-        if: needs.setup.outputs.DOING_RELEASE == '1'
-      - name: Copy target binaries
-        run: |
-          mkdir -p target/release
-          cp target/${{matrix.target}}/release/wasmer* target/release
-          cp target/${{matrix.target}}/release/libwasmer* target/release
-          if [ -d "wapm-cli" ]; then
-            mkdir -p wapm-cli/target/release 
-            cp wapm-cli/target/${{matrix.target}}/release/wapm* wapm-cli/target/release
-          fi
-        if: matrix.target
       - name: Dist
         run: |
           make distribution
@@ -243,7 +232,7 @@ jobs:
           make test-integration
         if: matrix.run_integration_tests && matrix.os != 'windows-latest'
       - name: Cross compile from Linux
-        if: matrix.os == 'ubuntu-18.04'
+        if: matrix.build == 'linux-x64'
         shell: bash
         run: |
           ls target/release
@@ -290,6 +279,64 @@ jobs:
           if-no-files-found: error
           retention-days: 1
 
+  test-docker:
+    name: Test on ${{ matrix.build }}
+    runs-on: ubuntu-latest
+    needs: setup
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - build: linux-musl-x64
+            image: alpine:latest
+            rust: 1.49
+            artifact_name: 'wasmer-linux-musl-amd64'
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: addnab/docker-run-action@v1
+        with:
+          image: ${{ matrix.image }}
+          options: -v ${{ github.workspace }}:/work
+          run: |
+            set -e
+
+            # Set up tools
+            apk add musl-dev curl make libtool libffi-dev gcc automake autoconf git
+
+            # Install Rust ${{ matrix.rust }}
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup.sh
+            sh ./rustup.sh -y
+            source ~/.cargo/env
+            rm -f ./rustup.sh
+
+            # Set up Rust
+            rustup toolchain install ${{ matrix.rust }}
+            rustup default ${{ matrix.rust }}
+
+            # Change working directory
+            cd /work
+
+            # Test
+            make test test-capi
+
+            # Build C API, Wasmer & Wapm
+            make build-capi build-wasmer build-wapm
+
+            # Dist
+            make distribution
+
+            # Run integration tests
+            export WASMER_DIR=`pwd`/package
+            make test-integration
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.artifact_name }}
+          path: dist
+          if-no-files-found: error
+          retention-days: 1
+
   test-cross-compile-on-linux:
     name: Test cross-compile on linux
     needs: [setup, test]
@@ -340,15 +387,14 @@ jobs:
   #        wasmer run artifacts/cross/cross_compiled_from_mac/win_from_mac.wjit -- --eval "function greet(name) { return JSON.stringify('Hello, ' + name); }; print(greet('World'));"
 
   release:
-    needs: [setup, test, test-cross-compile-on-linux, test-cross-compile-on-mac] #, test-cross-compile-on-win]
+    needs: [setup, test, test-docker, test-cross-compile-on-linux, test-cross-compile-on-mac] #, test-cross-compile-on-win]
     runs-on: ubuntu-latest
     if: needs.setup.outputs.DOING_RELEASE == '1'
     steps:
       - name: Download the Artifacts
         uses: actions/download-artifact@v2
         with:
           path: artifacts
-
       - name: Create Release
         id: create_release
         uses: actions/create-release@v1
@@ -359,7 +405,6 @@ jobs:
           release_name: Release ${{ needs.setup.outputs.VERSION }}
           draft: true
           prerelease: false
-
       - name: Upload Release Asset Windows Installer
         uses: actions/upload-release-asset@v1
         env:
@@ -369,7 +414,6 @@ jobs:
           asset_path: artifacts/wasmer-windows-amd64/WasmerInstaller.exe
           asset_name: wasmer-windows.exe
           asset_content_type: application/vnd.microsoft.portable-executable
-
       - name: Upload Release Asset Windows
         uses: actions/upload-release-asset@v1
         env:
@@ -379,7 +423,6 @@ jobs:
           asset_path: artifacts/wasmer-windows-amd64/wasmer.tar.gz
           asset_name: wasmer-windows-amd64.tar.gz
           asset_content_type: application/gzip
-
       - name: Upload Release Asset Linux amd64
         uses: actions/upload-release-asset@v1
         env:
@@ -389,7 +432,16 @@ jobs:
           asset_path: artifacts/wasmer-linux-amd64/wasmer.tar.gz
           asset_name: wasmer-linux-amd64.tar.gz
           asset_content_type: application/gzip
-
+      - name: Upload Release Asset Linux amd64 (musl)
+        id: upload-release-asset-linux-musl-amd64
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: artifacts/wasmer-linux-musl-amd64/wasmer.tar.gz
+          asset_name: wasmer-linux-musl-amd64.tar.gz
+          asset_content_type: application/gzip
       - name: Upload Release Asset Mac amd64
         uses: actions/upload-release-asset@v1
         env:
@@ -399,7 +451,6 @@ jobs:
           asset_path: artifacts/wasmer-darwin-amd64/wasmer.tar.gz
           asset_name: wasmer-darwin-amd64.tar.gz
           asset_content_type: application/gzip
-
       - name: Upload Release Asset Mac arm64
         uses: actions/upload-release-asset@v1
         env:
@@ -409,7 +460,6 @@ jobs:
           asset_path: artifacts/wasmer-darwin-arm64/wasmer.tar.gz
           asset_name: wasmer-darwin-arm64.tar.gz
           asset_content_type: application/gzip
-
       - name: Upload Release Asset Linux aarch64
         uses: actions/upload-release-asset@v1
         env:

diff --git a/Makefile b/Makefile
@@ -295,21 +295,34 @@ capi_compiler_features := --features $(subst $(space),$(comma),$(filter-out llvm
 ifneq (, $(filter 1, $(IS_DARWIN) $(IS_LINUX)))
 	bold := $(shell tput bold 2>/dev/null || echo -n '')
 	green := $(shell tput setaf 2 2>/dev/null || echo -n '')
+	yellow := $(shell tput setaf 3 2>/dev/null || echo -n '')
 	reset := $(shell tput sgr0 2>/dev/null || echo -n '')
 endif
 
 HOST_TARGET=$(shell rustup show | grep 'Default host: ' | cut -d':' -f2 | tr -d ' ')
 
+TARGET_DIR := target/release
+
+ifneq (, $(TARGET))
+	TARGET_DIR := target/$(TARGET)/release
+endif
+
 $(info -----------)
 $(info $(bold)$(green)INFORMATION$(reset))
 $(info -----------)
 $(info )
 $(info Host Target: `$(bold)$(green)$(HOST_TARGET)$(reset)`.)
+ifneq (, $(TARGET))
+	# We use spaces instead of tabs to indent `$(info)`
+	# otherwise it's considered as a command outside a
+	# target and it will fail.
+        $(info Build Target: $(bold)$(green)$(TARGET)$(reset) $(yellow)($(TARGET_DIR))$(reset))
+endif
 ifneq (, $(LIBC))
-		# We use spaces instead of tabs to indent `$(info)`
-		# otherwise it's considered as a command outside a
-		# target and it will fail.
-                $(info C standard library: $(bold)$(green)$(LIBC)$(reset))
+	# We use spaces instead of tabs to indent `$(info)`
+	# otherwise it's considered as a command outside a
+	# target and it will fail.
+        $(info C standard library: $(bold)$(green)$(LIBC)$(reset))
 endif
 $(info Enabled Compilers: $(bold)$(green)$(subst $(space),$(reset)$(comma)$(space)$(bold)$(green),$(compilers))$(reset).)
 $(info Compilers + engines pairs (for testing): $(bold)$(green)${compilers_engines}$(reset))
@@ -325,7 +338,6 @@ $(info --------------)
 $(info )
 $(info )
 
-
 ############
 # Building #
 ############
@@ -579,13 +591,13 @@ package-wapm:
 	mkdir -p "package/bin"
 ifneq (, $(filter 1, $(IS_DARWIN) $(IS_LINUX)))
 	if [ -d "wapm-cli" ]; then \
-		cp wapm-cli/target/release/wapm package/bin/ ;\
+		cp wapm-cli/$(TARGET_DIR)/wapm package/bin/ ;\
 		echo "#!/bin/bash\nwapm execute \"\$$@\"" > package/bin/wax ;\
 		chmod +x package/bin/wax ;\
 	fi
 else
 	if [ -d "wapm-cli" ]; then \
-		cp wapm-cli/target/release/wapm package/bin/ ;\
+		cp wapm-cli/$(TARGET_DIR)/wapm package/bin/ ;\
 	fi
 ifeq ($(IS_DARWIN), 1)
 	codesign -s - package/bin/wapm
@@ -606,9 +618,9 @@ endif
 package-wasmer:
 	mkdir -p "package/bin"
 ifeq ($(IS_WINDOWS), 1)
-	cp target/release/wasmer.exe package/bin/
+	cp $(TARGET_DIR)/wasmer.exe package/bin/
 else
-	cp target/release/wasmer package/bin/
+	cp $(TARGET_DIR)/wasmer package/bin/
 ifeq ($(IS_DARWIN), 1)
 	codesign -s - package/bin/wasmer
 endif
@@ -621,25 +633,28 @@ package-capi:
 	cp lib/c-api/wasmer_wasm.h* package/include
 	cp lib/c-api/wasm.h* package/include
 	cp lib/c-api/README.md package/include/README.md
-ifeq ($(IS_WINDOWS), 1)
-	cp target/release/wasmer_c_api.dll package/lib/wasmer.dll
-	cp target/release/wasmer_c_api.lib package/lib/wasmer.lib
-else
-ifeq ($(IS_DARWIN), 1)
+
+	# Windows
+	if [ -f $(TARGET_DIR)/wasmer_c_api.dll ]; then \
+		cp $(TARGET_DIR)/wasmer_c_api.dll package/lib/wasmer.dll ;\
+	fi
+	if [ -f $(TARGET_DIR)/wasmer_c_api.lib ]; then \
+		cp $(TARGET_DIR)/wasmer_c_api.lib package/lib/wasmer.lib ;\
+	fi
+
 	# For some reason in macOS arm64 there are issues if we copy constantly in the install_name_tool util
 	rm -f package/lib/libwasmer.dylib
-	cp target/release/libwasmer_c_api.dylib package/lib/libwasmer.dylib
-	cp target/release/libwasmer_c_api.a package/lib/libwasmer.a
-	# Fix the rpath for the dylib
-	install_name_tool -id "@rpath/libwasmer.dylib" package/lib/libwasmer.dylib
-else
-	# In some cases the .so may not be available, for example when building against musl (static linking)
-	if [ -f target/release/libwasmer_c_api.so ]; then \
-		cp target/release/libwasmer_c_api.so package/lib/libwasmer.so ;\
-	fi;
-	cp target/release/libwasmer_c_api.a package/lib/libwasmer.a
-endif
-endif
+	if [ -f $(TARGET_DIR)/libwasmer_c_api.dylib ]; then \
+		cp $(TARGET_DIR)/libwasmer_c_api.dylib package/lib/libwasmer.dylib ;\
+		install_name_tool -id "@rpath/libwasmer.dylib" package/lib/libwasmer.dylib ;\
+	fi
+
+	if [ -f $(TARGET_DIR)/libwasmer_c_api.so ]; then \
+		cp $(TARGET_DIR)/libwasmer_c_api.so package/lib/libwasmer.so ;\
+	fi
+	if [ -f $(TARGET_DIR)/libwasmer_c_api.a ]; then \
+		cp $(TARGET_DIR)/libwasmer_c_api.a package/lib/libwasmer.a ;\
+	fi
 
 package-docs: build-docs build-docs-capi
 	mkdir -p "package/docs/c"

diff --git a/bors.toml b/bors.toml
@@ -2,6 +2,7 @@ status = [
     "Audit",
     "Code lint",
     "Test on linux-x64",
+    "Test on linux-musl-x64",
     "Test on linux-aarch64",
     "Test on macos-x64",
     "Test on windows-x64",

diff --git a/lib/c-api/build.rs b/lib/c-api/build.rs
@@ -4,7 +4,10 @@
 //! * setting `inline-c` up.
 
 use cbindgen::{Builder, Language};
-use std::{env, fs, path::PathBuf};
+use std::{
+    env, fs,
+    path::{Path, PathBuf},
+};
 
 const PRE_HEADER: &'static str = r#"
 // Define the `ARCH_X86_X64` constant.
@@ -582,7 +585,17 @@ fn build_inline_c_env_vars() {
         } else if cfg!(target_os = "macos") {
             "libwasmer_c_api.dylib".to_string()
         } else {
-            "libwasmer_c_api.so".to_string()
+            let path = format!(
+                "{shared_object_dir}/{lib}",
+                shared_object_dir = shared_object_dir,
+                lib = "libwasmer_c_api.so"
+            );
+
+            if Path::new(path.as_str()).exists() {
+                "libwasmer_c_api.so".to_string()
+            } else {
+                "libwasmer_c_api.a".to_string()
+            }
         }
     );
 }
diff --git a/lib/c-api/tests/.gitignore b/lib/c-api/tests/.gitignore
@@ -1,6 +1,11 @@
 # ignore wasm-c-api binaries
 wasm-c-api-*
 test-*
+wasm-c-api/example/*
 
 # Unignore files ending with `.c` (i.e. `wasm-c-api-wasi.c`)
 !*.c
+!wasm-c-api/example/*.c
+!wasm-c-api/example/*.cc
+!wasm-c-api/example/*.wasm
+!wasm-c-api/example/*.wat