Merge branch 'master' into feature/public-api-wasi-fs

.appveyor.yml

@@ -18,24 +18,22 @@ environment:
 cache:
   - 'C:\Users\appveyor\.cargo'
   - target
-  - wapm-cli-target
 
 install:
-  # # Install LLVM
-  # - mkdir C:\projects\deps
-  # - cd C:\projects\deps
-  # - appveyor DownloadFile https://prereleases.llvm.org/win-snapshots/LLVM-8.0.0-r351033-win64.exe -FileName llvm.exe
-  # - 7z x llvm.exe -oC:\projects\deps\llvm
-  # # - set "PATH=%PATH%;C:\projects\deps\llvm\bin"
-  # - set "LLD_LINK=C:\projects\deps\llvm\bin\lld-link.exe"
-  # - set "LLVM_SYS_80_PREFIX=C:\projects\deps\llvm"
-  # - cd "%APPVEYOR_BUILD_FOLDER%"
+  # Install LLVM
+  - mkdir C:\projects\deps
+  - cd C:\projects\deps
+  - appveyor DownloadFile https://github.com/wasmerio/windows-llvm-build/releases/download/v8.0.0/llvm-8.0.0-install.zip -FileName llvm-8.0.0-install.zip
+  - 7z x llvm-8.0.0-install.zip
+  - C:\projects\deps\llvm-8.0.0-install\bin\llvm-config.exe --version
+  - set "LLVM_SYS_80_PREFIX=C:\projects\deps\llvm-8.0.0-install"
+  - cd "%APPVEYOR_BUILD_FOLDER%"
 
   # Install Rust
   # uncomment these lines if the cache is cleared, or if we must re-install rust for some reason
   # - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
   # - rustup-init.exe -yv --default-host %target%
-  - set PATH=%PATH%;C:\\Libraries\\llvm-5.0.0\\bin;%USERPROFILE%\.cargo\bin
+  - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
   - rustup default stable-%target%
   - rustup update
   - rustc -vV
@@ -52,16 +50,7 @@ build_script:
   - cargo build --release --verbose
   - git submodule init
   - git submodule update
-  # Cache wapm cli target in dir above to prevent breaking git submodule on windows
-  - if not exist wapm-cli-target mkdir wapm-cli-target
-  - move wapm-cli-target wapm-cli
-  - cd wapm-cli
-  - rename wapm-cli-target target
-  - cd ..
-  - cargo build --release --manifest-path wapm-cli/Cargo.toml --features "telemetry update-notifications"
-  - cd wapm-cli
-  - cd ..
-  - xcopy wapm-cli\target wapm-cli-target\ /i /y
+  - if %APPVEYOR_REPO_BRANCH%==master cargo build --release --manifest-path wapm-cli/Cargo.toml --features "telemetry update-notifications"
   - cargo build --release --manifest-path lib/runtime-c-api/Cargo.toml
 
 test_script:

.circleci/config.yml

@@ -115,6 +115,8 @@ jobs:
           name: Check
           command: |
             make check
+            make compile-bench-singlepass
+            # TODO: add compile-bench-llvm and compile-bench-clif when they work
       - run:
           name: Release
           command: make release-fast

CHANGELOG.md

@@ -6,6 +6,8 @@ Blocks of changes will separated by version increments.
 
 ## **[Unreleased]**
 - [#595](https://github.com/wasmerio/wasmer/pull/595) Add unstable public API for interfacing with the WASI file system in plugin-like usecases
+- [#598](https://github.com/wasmerio/wasmer/pull/598) LLVM Backend is now supported in Windows
+- [#599](https://github.com/wasmerio/wasmer/pull/599) Fix llvm backend failures in fat spec tests and simd_binaryen spec test.
 - [#579](https://github.com/wasmerio/wasmer/pull/579) Fix bug in caching with LLVM and Singlepass backends.
   Add `default-backend-singlepass`, `default-backend-llvm`, and `default-backend-cranelift` features to `wasmer-runtime`
   to control the `default_compiler()` function (this is a breaking change).  Add `compiler_for_backend` function in `wasmer-runtime`
@@ -29,6 +31,8 @@ Blocks of changes will separated by version increments.
   - makes wasitests-generate output stdout/stderr by default & adds function to print stdout and stderr for a command if it fails
   - compiles wasm with size optimizations & strips generated wasm with wasm-strip
 - [#554](https://github.com/wasmerio/wasmer/pull/554) Finish implementation of `wasi::fd_seek`, fix bug in filestat
+- [#550](https://github.com/wasmerio/wasmer/pull/550) Fix singlepass compilation error with `imul` instruction
+
 
 ## 0.5.5 - 2019-07-10
 - [#541](https://github.com/wasmerio/wasmer/pull/541) Fix dependency graph by making separate test crates; ABI implementations should not depend on compilers. Add Cranelift fork as git submodule of clif-backend
@@ -42,6 +46,7 @@ Blocks of changes will separated by version increments.
 - [#523](https://github.com/wasmerio/wasmer/pull/523) Update wapm version to fix bug related to signed packages in the global namespace and locally-stored public keys
 
 ## 0.5.2 - 2019-07-02
+- [#516](https://github.com/wasmerio/wasmer/pull/516) Add workaround for singlepass miscompilation on GetLocal
 - [#521](https://github.com/wasmerio/wasmer/pull/521) Update Wapm-cli, bump version numbers
 - [#518](https://github.com/wasmerio/wasmer/pull/518) Update Cranelift and WasmParser
 - [#514](https://github.com/wasmerio/wasmer/pull/514) [#519](https://github.com/wasmerio/wasmer/pull/519) Improved Emscripten network related calls, added a null check to `WasmPtr`

Cargo.toml

@@ -73,9 +73,23 @@ trace = ["wasmer-runtime-core/trace"]
 extra-debug = ["wasmer-clif-backend/debug", "wasmer-runtime-core/debug"]
 # This feature will allow cargo test to run much faster
 fast-tests = []
-backend-cranelift = ["wasmer-runtime-core/backend-cranelift", "wasmer-runtime/cranelift"]
-backend-llvm = ["wasmer-llvm-backend", "wasmer-runtime-core/backend-llvm", "wasmer-runtime/llvm"]
-backend-singlepass = ["wasmer-singlepass-backend", "wasmer-runtime-core/backend-singlepass", "wasmer-runtime/singlepass"]
+backend-cranelift = [
+    "wasmer-runtime-core/backend-cranelift",
+    "wasmer-runtime/cranelift",
+    "wasmer-middleware-common/clif"
+]
+backend-llvm = [
+    "wasmer-llvm-backend",
+    "wasmer-runtime-core/backend-llvm",
+    "wasmer-runtime/llvm",
+    "wasmer-middleware-common/llvm"
+]
+backend-singlepass = [
+    "wasmer-singlepass-backend",
+    "wasmer-runtime-core/backend-singlepass",
+    "wasmer-runtime/singlepass",
+    "wasmer-middleware-common/singlepass"
+]
 wasi = ["wasmer-wasi"]
 # vfs = ["wasmer-runtime-abi"]
 

Makefile

@@ -141,8 +141,20 @@ release-singlepass:
 release-llvm:
 	cargo build --release --features backend-llvm
 
-bench:
-	cargo bench --all
+bench-singlepass:
+	cargo bench --all --no-default-features --features "backend-singlepass"
+bench-clif:
+	cargo bench --all --no-default-features --features "backend-clif"
+bench-llvm:
+	cargo bench --all --no-default-features --features "backend-llvm"
+
+# compile but don't run the benchmarks
+compile-bench-singlepass:
+	cargo bench --all --no-run --no-default-features --features "backend-singlepass"
+compile-bench-clif:
+	cargo bench --all --no-run --no-default-features --features "backend-clif"
+compile-bench-llvm:
+	cargo bench --all --no-run --no-default-features --features "backend-llvm"
 
 
 # Build utils

README.md

@@ -220,7 +220,10 @@ Each integration can be tested separately:
 Benchmarks can be run with:
 
 ```sh
-make bench
+make bench-[backend]
+
+# for example
+make bench-singlepass
 ```
 
 ## Roadmap

lib/llvm-backend/Cargo.toml

@@ -1,13 +1,12 @@
 [package]
 name = "wasmer-llvm-backend"
 version = "0.5.7"
-authors = ["Lachlan Sneff <[email protected]>"]
+authors = ["The Wasmer Engineering Team <[email protected]>"]
 edition = "2018"
 readme = "README.md"
 
 [dependencies]
 wasmer-runtime-core = { path = "../runtime-core", version = "0.5.7" }
-inkwell = { git = "https://github.com/wasmerio/inkwell", branch = "llvm8-0", features = ["llvm8-0"] }
 wasmparser = "0.34.0"
 hashbrown = "0.1.8"
 smallvec = "0.6.8"
@@ -16,6 +15,15 @@ libc = "0.2.49"
 nix = "0.14.0"
 capstone = { version = "0.5.0", optional = true }
 
+[dependencies.inkwell]
+git = "https://github.com/wasmerio/inkwell"
+branch = "llvm8-0"
+default-features = false
+features = ["llvm8-0", "target-x86"]
+
+[target.'cfg(windows)'.dependencies]
+winapi = { version = "0.3", features = ["memoryapi"] }
+
 [build-dependencies]
 cc = "1.0"
 lazy_static = "1.2.0"

lib/llvm-backend/cpp/object_loader.cpp

@@ -75,16 +75,24 @@ struct MemoryManager : llvm::RuntimeDyld::MemoryManager {
     }
 
     virtual void registerEHFrames(uint8_t* addr, uint64_t LoadAddr, size_t size) override {
+        // We don't know yet how to do this on Windows, so we hide this on compilation
+        // so we can compile and pass spectests on unix systems
+        #ifndef _WIN32
         eh_frame_ptr = addr;
         eh_frame_size = size;
         eh_frames_registered = true;
         callbacks.visit_fde(addr, size, __register_frame);
+        #endif
     }
 
     virtual void deregisterEHFrames() override {
+        // We don't know yet how to do this on Windows, so we hide this on compilation
+        // so we can compile and pass spectests on unix systems
+        #ifndef _WIN32
         if (eh_frames_registered) {
             callbacks.visit_fde(eh_frame_ptr, eh_frame_size, __deregister_frame);
         }
+        #endif
     }
 
     virtual bool finalizeMemory(std::string *ErrMsg = nullptr) override {

lib/llvm-backend/src/backend.rs

@@ -1,14 +1,12 @@
 use crate::intrinsics::Intrinsics;
+use crate::structs::{Callbacks, LLVMModule, LLVMResult, MemProtect};
 use inkwell::{
     memory_buffer::MemoryBuffer,
     module::Module,
     targets::{CodeModel, FileType, InitializationConfig, RelocMode, Target, TargetMachine},
     OptimizationLevel,
 };
-use libc::{
-    c_char, mmap, mprotect, munmap, MAP_ANON, MAP_PRIVATE, PROT_EXEC, PROT_NONE, PROT_READ,
-    PROT_WRITE,
-};
+use libc::c_char;
 use std::{
     any::Any,
     ffi::{c_void, CString},
@@ -31,42 +29,6 @@ use wasmer_runtime_core::{
     vm, vmcalls,
 };
 
-#[repr(C)]
-struct LLVMModule {
-    _private: [u8; 0],
-}
-
-#[allow(non_camel_case_types, dead_code)]
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-#[repr(C)]
-enum MemProtect {
-    NONE,
-    READ,
-    READ_WRITE,
-    READ_EXECUTE,
-}
-
-#[allow(non_camel_case_types, dead_code)]
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-#[repr(C)]
-enum LLVMResult {
-    OK,
-    ALLOCATE_FAILURE,
-    PROTECT_FAILURE,
-    DEALLOC_FAILURE,
-    OBJECT_LOAD_FAILURE,
-}
-
-#[repr(C)]
-struct Callbacks {
-    alloc_memory: extern "C" fn(usize, MemProtect, &mut *mut u8, &mut usize) -> LLVMResult,
-    protect_memory: extern "C" fn(*mut u8, usize, MemProtect) -> LLVMResult,
-    dealloc_memory: extern "C" fn(*mut u8, usize) -> LLVMResult,
-
-    lookup_vm_symbol: extern "C" fn(*const c_char, usize) -> *const vm::Func,
-    visit_fde: extern "C" fn(*mut u8, usize, extern "C" fn(*mut u8)),
-}
-
 extern "C" {
     fn module_load(
         mem_ptr: *const u8,
@@ -99,69 +61,21 @@ extern "C" {
 }
 
 fn get_callbacks() -> Callbacks {
-    fn round_up_to_page_size(size: usize) -> usize {
-        (size + (4096 - 1)) & !(4096 - 1)
-    }
-
     extern "C" fn alloc_memory(
         size: usize,
         protect: MemProtect,
         ptr_out: &mut *mut u8,
         size_out: &mut usize,
     ) -> LLVMResult {
-        let size = round_up_to_page_size(size);
-        let ptr = unsafe {
-            mmap(
-                ptr::null_mut(),
-                size,
-                match protect {
-                    MemProtect::NONE => PROT_NONE,
-                    MemProtect::READ => PROT_READ,
-                    MemProtect::READ_WRITE => PROT_READ | PROT_WRITE,
-                    MemProtect::READ_EXECUTE => PROT_READ | PROT_EXEC,
-                },
-                MAP_PRIVATE | MAP_ANON,
-                -1,
-                0,
-            )
-        };
-        if ptr as isize == -1 {
-            return LLVMResult::ALLOCATE_FAILURE;
-        }
-        *ptr_out = ptr as _;
-        *size_out = size;
-        LLVMResult::OK
+        unsafe { crate::platform::alloc_memory(size, protect, ptr_out, size_out) }
     }
 
     extern "C" fn protect_memory(ptr: *mut u8, size: usize, protect: MemProtect) -> LLVMResult {
-        let res = unsafe {
-            mprotect(
-                ptr as _,
-                round_up_to_page_size(size),
-                match protect {
-                    MemProtect::NONE => PROT_NONE,
-                    MemProtect::READ => PROT_READ,
-                    MemProtect::READ_WRITE => PROT_READ | PROT_WRITE,
-                    MemProtect::READ_EXECUTE => PROT_READ | PROT_EXEC,
-                },
-            )
-        };
-
-        if res == 0 {
-            LLVMResult::OK
-        } else {
-            LLVMResult::PROTECT_FAILURE
-        }
+        unsafe { crate::platform::protect_memory(ptr, size, protect) }
     }
 
     extern "C" fn dealloc_memory(ptr: *mut u8, size: usize) -> LLVMResult {
-        let res = unsafe { munmap(ptr as _, round_up_to_page_size(size)) };
-
-        if res == 0 {
-            LLVMResult::OK
-        } else {
-            LLVMResult::DEALLOC_FAILURE
-        }
+        unsafe { crate::platform::dealloc_memory(ptr, size) }
     }
 
     extern "C" fn lookup_vm_symbol(name_ptr: *const c_char, length: usize) -> *const vm::Func {