From 8e144a457b7eec347b2f4d6c58125d4cb3cfae77 Mon Sep 17 00:00:00 2001 From: Syrus Date: Mon, 30 Mar 2020 14:18:36 -0700 Subject: [PATCH 1/6] Adding Azure pipelines to handle ARM --- .azure/install-llvm.yml | 18 ++++++++- .azure/install-sccache.yml | 10 +++-- .travis.yml | 75 -------------------------------------- azure-pipelines.yml | 16 +++++--- 4 files changed, 34 insertions(+), 85 deletions(-) delete mode 100644 .travis.yml diff --git a/.azure/install-llvm.yml b/.azure/install-llvm.yml index 3b9ffb211ac..2d89f835129 100644 --- a/.azure/install-llvm.yml +++ b/.azure/install-llvm.yml @@ -26,8 +26,22 @@ steps: echo "##vso[task.prependpath]$LLVM_PATH/bin" echo "##vso[task.setvariable variable=LLVM_SYS_80_PREFIX]$LLVM_PATH" fi - displayName: "Install LLVM (Linux)" - condition: eq(variables['Agent.OS'], 'Linux') + displayName: "Install LLVM (Linux X64)" + condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OSArchitecture'], 'X64')) + + - bash: | + set -ex + if [ -x "`command -v llvm-config`" ]; then + echo `command -v llvm-config` `llvm-config --version` installed + else + curl -O https://releases.llvm.org/8.0.0/clang+llvm-8.0.0-aarch64-linux-gnu.tar.xz + tar xf clang+llvm-8.0.0-aarch64-linux-gnu.tar.xz + export LLVM_PATH="`pwd`/clang+llvm-8.0.0-aarch64-linux-gnu/" + echo "##vso[task.prependpath]$LLVM_PATH/bin" + echo "##vso[task.setvariable variable=LLVM_SYS_80_PREFIX]$LLVM_PATH" + fi + displayName: "Install LLVM (Linux ARM)" + condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OSArchitecture'], 'ARM64')) - bash: | set -ex diff --git a/.azure/install-sccache.yml b/.azure/install-sccache.yml index 091dcc572b0..3a158bc3fad 100644 --- a/.azure/install-sccache.yml +++ b/.azure/install-sccache.yml @@ -9,8 +9,12 @@ steps: curl -L https://github.com/mozilla/sccache/releases/download/0.2.10/sccache-0.2.10-x86_64-unknown-linux-musl.tar.gz | tar xzf - sccache=`pwd`/sccache-0.2.10-x86_64-unknown-linux-musl/sccache echo "##vso[task.setvariable variable=RUSTC_WRAPPER;]$sccache" - displayName: Install sccache - Linux - condition: eq( variables['Agent.OS'], 'Linux' ) + displayName: Install sccache - Linux X64 + condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OSArchitecture'], 'X64')) + # - bash: | + # echo "##vso[task.setvariable variable=RUSTC_WRAPPER;]sccache" + # displayName: Install sccache - Linux ARM + # condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OSArchitecture'], 'ARM64')) - bash: | set -ex brew install openssl@1.1 @@ -32,7 +36,7 @@ steps: $RUSTC_WRAPPER -s cat sccache.log displayName: "start sccache" - condition: not(eq( variables['Agent.OS'], 'Windows_NT' )) + condition: not(or(eq( variables['Agent.OS'], 'Windows_NT' ), eq(variables['Agent.OSArchitecture'], 'ARM64'))) env: SCCACHE_AZURE_CONNECTION_STRING: $(SCCACHE_AZURE_CONNECTION_STRING) SCCACHE_AZURE_BLOB_CONTAINER: $(SCCACHE_AZURE_BLOB_CONTAINER) diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index cda6825ee75..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,75 +0,0 @@ -arch: - - arm64 - -language: rust -install: travis_retry -rust: - - nightly-2019-12-19 - -cache: - directories: - - /home/travis/.sccache/ - - /home/travis/.cargo/bin/ - -script: - # Sccache - # - curl -L https://github.com/mozilla/sccache/releases/download/0.2.10/sccache-0.2.10-x86_64-unknown-linux-musl.tar.gz | tar xzf - - # - export RUSTC_WRAPPER=`pwd`/sccache-0.2.10-x86_64-unknown-linux-musl/sccache - - test -f /home/travis/.cargo/bin/sccache || travis_retry cargo install sccache - - export RUSTC_WRAPPER=/home/travis/.cargo/bin/sccache - - mkdir -p /home/travis/.sccache/ - - export SCCACHE_DIR="/home/travis/.sccache/" - - SCCACHE_ERROR_LOG=`pwd`/sccache.log RUST_LOG=debug $RUSTC_WRAPPER --start-server - - $RUSTC_WRAPPER -s - - # Tests - - make spectests-singlepass - -before_deploy: - # Release - - make release-singlepass - - mkdir -p artifacts - # Make capi - ## Disable capi tests for now: - ## They are failing because trampolines are not yet implemented for ARM - # - make test-capi-singlepass - - make capi-singlepass - - make build-capi-package - - cp ./wasmer-c-api.tar.gz ./artifacts/$(./scripts/capi-name.sh) - # Build WAPM - - make build-wapm - # Make package - - make build-install-package - - cp ./wasmer.tar.gz ./artifacts/$(./scripts/binary-name.sh) - -# before_deploy: -# # Set up git user name and tag this commit -# - git config --local user.name "Syrus Akbary" -# - git config --local user.email "syrus@wasmer.io" -# - export TRAVIS_TAG="0.10.2" -# # - git tag $TRAVIS_TAG - -deploy: - provider: releases - file_glob: true - file: artifacts/* - api_key: $GITHUB_OAUTH_TOKEN - # This is set to the previous artifacts are not deleted by travis - skip_cleanup: true - on: - tags: true - # branch: feature/singlepass-aarch64 - -addons: - apt: - packages: - - cmake - -branches: - only: - - master - - staging - - trying - # Making sure Travis runs on new Tags - - /^\d+\.\d+(\.\d+)?(-\S*)?$/ - - /^v\d+\.\d+(\.\d+)?(-\S*)?$/ diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8665c1dba3a..7aae3647667 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -81,7 +81,10 @@ jobs: condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) - bash: make test displayName: Tests (*nix) - condition: and(succeeded(), not(eq(variables['Agent.OS'], 'Windows_NT'))) + condition: and(succeeded(), eq(variables['Agent.OSArchitecture'], 'X64'), not(eq(variables['Agent.OS'], 'Windows_NT'))) + - bash: make spectests-singlepass + displayName: Tests (Linux ARM) + condition: and(succeeded(), eq(variables['Agent.OSArchitecture'], 'ARM64')) - bash: make spectests-cranelift displayName: Tests (Windows) condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) @@ -134,11 +137,14 @@ jobs: mkdir -p artifacts displayName: Create Artifacts Dir - bash: make release - displayName: Build (*nix) - condition: and(succeeded(), not(eq(variables['Agent.OS'], 'Windows_NT'))) + displayName: Build (*nix X64) + condition: and(succeeded(), not(eq(variables['Agent.OS'], 'Windows_NT')), eq(variables['Agent.OSArchitecture'], 'X64')) - bash: sudo apt-get install musl-tools && make release-musl displayName: Build (Linux, x86_64-unknown-linux-musl) - condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OSArchitecture'], 'X64')) + - bash: make release-singlepass + displayName: Build (ARM) + condition: and(succeeded(), not(eq(variables['Agent.OS'], 'Windows_NT')), eq(variables['Agent.OSArchitecture'], 'ARM64')) - bash: make release-llvm displayName: Build (Windows) condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) @@ -204,7 +210,7 @@ jobs: - bash: | make test-capi displayName: Test c-api - condition: and(succeeded(), not(eq(variables['Agent.OS'], 'Windows_NT'))) + condition: and(succeeded(), eq(variables['Agent.OSArchitecture'], 'X64'), not(eq(variables['Agent.OS'], 'Windows_NT'))) - bash: | make capi displayName: Build c-api From eec7844bae2c792e061c7bb9cbb5bbe362b99049 Mon Sep 17 00:00:00 2001 From: Syrus Date: Mon, 30 Mar 2020 14:45:04 -0700 Subject: [PATCH 2/6] Added ARM integration --- azure-pipelines.yml | 31 ++++++++++++++----------------- bors.toml | 3 +-- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2b93a027758..cf4ca2a3ed2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -44,6 +44,7 @@ jobs: strategy: matrix: linux: + poolName: "Azure Pipelines" imageName: "ubuntu-16.04" rust_toolchain: nightly-2019-12-19 android: @@ -51,16 +52,22 @@ jobs: rust_toolchain: nightly-2019-12-19 ANDROID: true mac: + poolName: "Azure Pipelines" imageName: "macos-10.14" rust_toolchain: nightly-2019-12-19 # By default schannel checks revocation of certificates unlike some other SSL # backends, but we've historically had problems on CI where a revocation # server goes down presumably. See #43333 for more info CARGO_HTTP_CHECK_REVOKE: false + arm: + poolName: "Packet" + rust_toolchain: nightly-2019-12-19 windows: + poolName: "Azure Pipelines" imageName: "vs2017-win2016" rust_toolchain: '1.41.1' pool: + name: $(poolName) vmImage: $(imageName) condition: in(variables['Build.SourceBranch'], 'refs/heads/master', 'refs/heads/staging', 'refs/heads/trying') steps: @@ -116,13 +123,19 @@ jobs: strategy: matrix: linux: + poolName: "Azure Pipelines" imageName: "ubuntu-16.04" rust_toolchain: nightly-2019-12-19 mac: + poolName: "Azure Pipelines" imageName: "macos-10.14" rust_toolchain: nightly-2019-12-19 MACOSX_DEPLOYMENT_TARGET: 10.10 + arm: + poolName: "Packet" + rust_toolchain: nightly-2019-12-19 windows: + poolName: "Azure Pipelines" imageName: "vs2017-win2016" rust_toolchain: '1.41.1' # RUSTFLAGS: -Ctarget-feature=+crt-static @@ -283,9 +296,8 @@ jobs: - script: VERSION_TAG=`git describe --tags` && echo "##vso[task.setvariable variable=VERSION_TAG]$VERSION_TAG" displayName: Set the tag name as an environment variable - task: GithubRelease@0 - displayName: "Create GitHub Release" + displayName: "Create GitHub Release with artifacts" condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags')) - continueOnError: true inputs: gitHubConnection: 'wasmer' repositoryName: 'wasmerio/wasmer' @@ -297,21 +309,6 @@ jobs: # TODO: automate it all by getting the release notes from somewhere else and using the `releaseNotesFile` key isDraft: false isPreRelease: false - - task: GithubRelease@0 - displayName: "Update GitHub Release with assets" - condition: and(succeededOrFailed(), startsWith(variables['Build.SourceBranch'], 'refs/tags')) - inputs: - gitHubConnection: 'wasmer' - repositoryName: 'wasmerio/wasmer' - action: 'edit' - target: '$(Build.SourceVersion)' - title: '$(VERSION_TAG)' - tag: $(VERSION_TAG) - addChangeLog: false - tagSource: 'auto' - # TODO: automate it all by getting the release notes from somewhere else and using the `releaseNotesFile` key - isDraft: false - isPreRelease: false assets: '$(Build.ArtifactStagingDirectory)/**' assetUploadMode: 'replace' # Don't delete previously uploaded assets (default) diff --git a/bors.toml b/bors.toml index 936db84337a..1003d9678c1 100644 --- a/bors.toml +++ b/bors.toml @@ -1,6 +1,5 @@ status = [ - "wasmerio.wasmer", - "continuous-integration/travis-ci/push" + "wasmerio.wasmer" ] required_approvals = 1 timeout_sec = 7200 From d5cdd497faa78d00f9f57f0b0233558a3d23f654 Mon Sep 17 00:00:00 2001 From: Syrus Date: Mon, 30 Mar 2020 15:03:50 -0700 Subject: [PATCH 3/6] Fixed missing poolName --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cf4ca2a3ed2..ad2037faef9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -140,6 +140,7 @@ jobs: rust_toolchain: '1.41.1' # RUSTFLAGS: -Ctarget-feature=+crt-static pool: + name: $(poolName) vmImage: $(imageName) condition: | or( From 18c0896423dc93128c294c82a6ae98b600d19d7a Mon Sep 17 00:00:00 2001 From: Syrus Date: Mon, 30 Mar 2020 15:10:31 -0700 Subject: [PATCH 4/6] Trying to ffix pipelines --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ad2037faef9..9a3a85e9341 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -48,6 +48,7 @@ jobs: imageName: "ubuntu-16.04" rust_toolchain: nightly-2019-12-19 android: + poolName: "Azure Pipelines" imageName: "ubuntu-16.04" rust_toolchain: nightly-2019-12-19 ANDROID: true From eb4dc7120808d0da22a96e3e50bb46444341cd9e Mon Sep 17 00:00:00 2001 From: Syrus Date: Mon, 30 Mar 2020 15:28:47 -0700 Subject: [PATCH 5/6] Added changes in changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecc329667c9..b63f91207a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## **[Unreleased]** +- [#1345](https://github.com/wasmerio/wasmer/pull/1345) Adding ARM testing in Azure Pipelines - [#1335](https://github.com/wasmerio/wasmer/pull/1335) Change mutability of `memory` to `const` in `wasmer_memory_data_length` in the C API - [#1329](https://github.com/wasmerio/wasmer/pull/1329) New numbers and strings instructions for WIT - [#1332](https://github.com/wasmerio/wasmer/pull/1332) Add option to `CompilerConfig` to force compiler IR verification off even when `debug_assertions` are enabled. This can be used to make debug builds faster, which may be important if you're creating a library that wraps Wasmer and depend on the speed of debug builds. From 800d428e8cadf36866f010cd5b62e9d5ddf170be Mon Sep 17 00:00:00 2001 From: Syrus Date: Mon, 30 Mar 2020 15:36:42 -0700 Subject: [PATCH 6/6] Added the architecture into the artifacts --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9a3a85e9341..e439e09bc2c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -197,7 +197,7 @@ jobs: eq(variables['Agent.OS'], 'Windows_NT') ) - publish: $(System.DefaultWorkingDirectory)/artifacts - artifact: cli-$(Agent.OS) + artifact: cli-$(Agent.OS)-$(Agent.OSArchitecture) - job: Build_Library strategy: @@ -255,7 +255,7 @@ jobs: eq(variables['Agent.OS'], 'Windows_NT') ) - publish: $(System.DefaultWorkingDirectory)/artifacts - artifact: library-$(Agent.OS) + artifact: library-$(Agent.OS)-$(Agent.OSArchitecture) - job: Build_Docs pool: