Proof/rebase/swiftwasm/2020 01 12

.github/workflows/main.yml

@@ -0,0 +1,137 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - swiftwasm
+  pull_request:
+    branches:
+      - swiftwasm
+
+jobs:
+  linux_build:
+    timeout-minutes: 0
+    runs-on: ubuntu-18.04
+
+    steps:
+      - uses: actions/checkout@v1
+      - name: Build Linux installable archive
+        run: ./utils/webassembly/ci-linux.sh
+      - name: Upload Linux installable archive
+        uses: actions/upload-artifact@v1
+        with:
+          name: linux-installable
+          path: ../swiftwasm-linux.tar.gz
+      - name: Pack test results
+        run: tar cJf swift-test-results.tar.gz ../build/*/swift-linux-x86_64/swift-test-results
+      - name: Upload test results
+        uses: actions/upload-artifact@v1
+        with:
+          name: linux-test-results
+          path: ./swift-test-results.tar.gz
+
+  macos_build:
+    timeout-minutes: 0
+    runs-on: macos-latest
+
+    steps:
+      - uses: actions/checkout@v1
+      - name: Build macOS installable archive
+        run: ./utils/webassembly/ci-mac.sh
+      - name: Upload macOS installable archive
+        uses: actions/upload-artifact@v1
+        with:
+          name: macos-installable
+          path: ../swiftwasm-macos.tar.gz
+      - name: Upload packaging scripts
+        uses: actions/upload-artifact@v1
+        with:
+          name: packaging-scripts
+          path: utils/webassembly
+      - name: Pack test results
+        run: tar cJf swift-test-results.tar.gz ../build/*/swift-macosx-x86_64/swift-test-results
+      - name: Upload test results
+        uses: actions/upload-artifact@v1
+        with:
+          name: macos-test-results
+          path: ./swift-test-results.tar.gz
+  package:
+    name: Build SwiftWasm packages
+    needs:
+      - linux_build
+      - macos_build
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Download installable Linux archive
+        uses: actions/download-artifact@v1
+        with:
+          name: linux-installable
+      - name: Download installable macOS archive
+        uses: actions/download-artifact@v1
+        with:
+          name: macos-installable
+      - name: Download packaging scripts
+        uses: actions/download-artifact@v1
+        with:
+          name: packaging-scripts
+      - name: Build the packages
+        shell: bash
+        run: |
+          cd packaging-scripts
+          find . -name '*.sh' -exec chmod +x {} \;
+          chmod +x sdkroot/swiftwasm
+          ./download-prebuilts.sh
+
+          cp ../linux-installable/swiftwasm-linux.tar.gz \
+            ../macos-installable/swiftwasm-macos.tar.gz \
+            prebuilt
+          ./build-packages.sh
+
+          cd output
+          tar xf swiftwasm-sdk-linux.tar.xz && echo "Successfully unpacked Linux SDK"
+
+          cd swiftwasm-sdk
+          ./swiftwasm example/hello.swift hello.wasm && echo "Successfully linked hello.wasm"
+
+      - name: Upload macOS package
+        uses: actions/upload-artifact@v1
+        with:
+          name: macos-package
+          path: packaging-scripts/output/swiftwasm-sdk-macos.tar.xz
+
+      - name: Upload Linux package
+        uses: actions/upload-artifact@v1
+        with:
+          name: linux-package
+          path: packaging-scripts/output/swiftwasm-sdk-linux.tar.xz
+
+      - name: Upload hello.wasm compiled with Linux package
+        uses: actions/upload-artifact@v1
+        with:
+          name: linux-hello.wasm
+          path: packaging-scripts/output/swiftwasm-sdk/hello.wasm
+
+  macos_smoke_test:
+    name: Compile hello.swift on macOS
+    runs-on: macos-latest
+    needs: package
+    steps:
+      - name: Download SwiftWasm macOS package
+        uses: actions/download-artifact@v1
+        with:
+          name: macos-package
+
+      - name: Build hello.wasm
+        shell: bash
+        run: |
+          cd macos-package
+          tar xf swiftwasm-sdk-macos.tar.xz && echo "Successfully unpacked macOS SDK"
+
+          cd swiftwasm-sdk
+          ./swiftwasm example/hello.swift hello.wasm && echo "Successfully linked hello.wasm"
+
+      - name: Upload hello.wasm compiled with macOS package
+        uses: actions/upload-artifact@v1
+        with:
+          name: macos-hello.wasm
+          path: macos-package/swiftwasm-sdk/hello.wasm

.gitignore

@@ -39,6 +39,9 @@ docs/_build
 # Visual Studio metadata
 .vs
 
+# clangd
+.clangd
+
 #==============================================================================#
 # Ignore CMake temporary files
 #==============================================================================#

.mailmap

@@ -8,6 +8,7 @@ Amr Aboelela <amraboelela@gmail.com> <amraboelela@users.noreply.github.com>
 Ankit Aggarwal <ankit_aggarwal@apple.com> <ankit.spd@gmail.com>
 Argyrios Kyrtzidis <kyrtzidis@apple.com> <akyrtzi@gmail.com>
 Arsen Gasparyan <to.arsen.gasparyan@gmail.com> <frootloops@users.noreply.github.com>
+Ashley Garland <acgarland@apple.com> <dfarler@apple.com>
 Ben Cohen <ben_cohen@apple.com>
 Ben Cohen <ben_cohen@apple.com> <airspeedswift@users.noreply.github.com>
 Ben Cohen <ben_cohen@apple.com> <ben@airspeedvelocity.net>

CMakeLists.txt

@@ -255,11 +255,11 @@ set(SWIFT_ANDROID_DEPLOY_DEVICE_PATH "" CACHE STRING
   "Path on an Android device where build products will be pushed. These are used when running the test suite against the device")
 
 #
-# User-configurable ICU specific options for Android, FreeBSD, Linux and Haiku.
+# User-configurable ICU specific options for Android, FreeBSD, Linux, Haiku, and WebAssembly.
 #
 
-foreach(sdk ANDROID;FREEBSD;LINUX;WINDOWS;HAIKU)
-  foreach(arch aarch64;armv6;armv7;i686;powerpc64;powerpc64le;s390x;x86_64)
+foreach(sdk ANDROID;FREEBSD;LINUX;WINDOWS;HAIKU;WASI)
+  foreach(arch aarch64;armv6;armv7;i686;powerpc64;powerpc64le;s390x;wasm32;x86_64)
     set(SWIFT_${sdk}_${arch}_ICU_UC "" CACHE STRING
         "Path to a directory containing the icuuc library for ${sdk}")
     set(SWIFT_${sdk}_${arch}_ICU_UC_INCLUDE "" CACHE STRING
@@ -633,6 +633,8 @@ else()
     set(SWIFT_HOST_VARIANT_ARCH_default "powerpc64le")
   elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "s390x")
     set(SWIFT_HOST_VARIANT_ARCH_default "s390x")
+  elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "wasm32")
+    set(SWIFT_HOST_VARIANT_ARCH_default "wasm32")
   # FIXME: Only matches v6l/v7l - by far the most common variants
   elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "armv6l")
     set(SWIFT_HOST_VARIANT_ARCH_default "armv6")
@@ -817,6 +819,18 @@ if(swift_build_windows AND NOT "${CMAKE_SYSTEM_NAME}" STREQUAL "Windows")
   configure_sdk_windows("Windows" "msvc" "${SWIFT_SDK_WINDOWS_ARCHITECTURES}")
 endif()
 
+# Should we cross-compile the standard library for WebAssembly (WASI)?
+is_sdk_requested(WASI swift_build_wasm)
+if(swift_build_wasm AND NOT "${SWIFT_HOST_VARIANT_SDK}" STREQUAL "WASI")
+  if(SWIFT_BUILD_DYNAMIC_SDK_OVERLAY OR SWIFT_BUILD_DYNAMIC_STDLIB)
+    message(FATAL_ERROR "Unable to build dynamic overlay/stdlib for WASI. WASM does not support dynamic linking.")
+  endif()
+  if("${SWIFT_SDK_WASI_ARCHITECTURES}" STREQUAL "")
+    set(SWIFT_SDK_WASI_ARCHITECTURES wasm32)
+  endif()
+  configure_sdk_unix("WASI" "${SWIFT_SDK_WASI_ARCHITECTURES}")
+endif()
+
 if("${SWIFT_SDKS}" STREQUAL "")
   set(SWIFT_SDKS "${SWIFT_CONFIGURED_SDKS}")
 endif()
@@ -924,14 +938,24 @@ else()
   find_package(LibXml2)
 endif()
 
-# You need libedit linked in order to check if you have el_wgets.
-cmake_push_check_state()
-list(APPEND CMAKE_REQUIRED_LIBRARIES "edit")
-check_symbol_exists(el_wgets "histedit.h" HAVE_EL_WGETS)
-if(HAVE_EL_WGETS)
-  set(HAVE_UNICODE_LIBEDIT 1)
+if(LLVM_ENABLE_LIBEDIT)
+  find_package(LibEdit REQUIRED)
+else()
+  find_package(LibEdit)
+endif()
+
+if(LibEdit_FOUND)
+  cmake_push_check_state()
+  list(APPEND CMAKE_REQUIRED_INCLUDES ${LibEdit_INCLUDE_DIRS})
+  list(APPEND CMAKE_REQUIRED_LIBRARIES ${LibEdit_LIBRARIES})
+  check_symbol_exists(el_wgets "histedit.h" HAVE_EL_WGETS)
+  if(HAVE_EL_WGETS)
+    set(LibEdit_HAS_UNICODE YES)
+  else()
+    set(LibEdit_HAS_UNICODE NO)
+  endif()
+  cmake_pop_check_state()
 endif()
-cmake_pop_check_state()
 
 check_symbol_exists(wait4 "sys/wait.h" HAVE_WAIT4)
 
@@ -953,10 +977,16 @@ if(SWIFT_BUILD_SYNTAXPARSERLIB OR SWIFT_BUILD_SOURCEKIT)
       set(SWIFT_LIBDISPATCH_CXX_COMPILER ${CMAKE_CXX_COMPILER})
     elseif(${CMAKE_SYSTEM_NAME} STREQUAL ${CMAKE_HOST_SYSTEM_NAME})
       if(CMAKE_SYSTEM_NAME STREQUAL Windows)
-        set(SWIFT_LIBDISPATCH_C_COMPILER
-            $<TARGET_FILE_DIR:clang>/clang-cl${CMAKE_EXECUTABLE_SUFFIX})
-        set(SWIFT_LIBDISPATCH_CXX_COMPILER
-            $<TARGET_FILE_DIR:clang>/clang-cl${CMAKE_EXECUTABLE_SUFFIX})
+        if(CMAKE_SYSTEM_PROCESSOR STREQUAL CMAKE_HOST_SYSTEM_PROCESSOR AND
+            TARGET clang)
+          set(SWIFT_LIBDISPATCH_C_COMPILER
+              $<TARGET_FILE_DIR:clang>/clang-cl${CMAKE_EXECUTABLE_SUFFIX})
+          set(SWIFT_LIBDISPATCH_CXX_COMPILER
+              $<TARGET_FILE_DIR:clang>/clang-cl${CMAKE_EXECUTABLE_SUFFIX})
+        else()
+          set(SWIFT_LIBDISPATCH_C_COMPILER clang-cl${CMAKE_EXECUTABLE_SUFFIX})
+          set(SWIFT_LIBDISPATCH_CXX_COMPILER clang-cl${CMAKE_EXECUTABLE_SUFFIX})
+        endif()
       else()
         set(SWIFT_LIBDISPATCH_C_COMPILER $<TARGET_FILE_DIR:clang>/clang)
         set(SWIFT_LIBDISPATCH_CXX_COMPILER $<TARGET_FILE_DIR:clang>/clang++)

README.md

@@ -86,7 +86,7 @@ Please make sure you use Python 2.x. Python 3.x is not supported currently.
 
 #### macOS
 
-To build for macOS, you need [Xcode 11.2](https://developer.apple.com/xcode/downloads/).
+To build for macOS, you need [Xcode 11.3](https://developer.apple.com/xcode/downloads/).
 The required version of Xcode changes frequently, and is often a beta release.
 Check this document or the host information on <https://ci.swift.org> for the
 current required version.

benchmark/scripts/Benchmark_DTrace.in

@@ -70,7 +70,7 @@ class DTraceBenchmarkDriver(perf_test_driver.BenchmarkDriver):
     def __init__(self, binary, xfail_list, csv_output):
         perf_test_driver.BenchmarkDriver.__init__(
             self, binary, xfail_list,
-            enable_parallel=False,
+            enable_parallel=True,
             opt_levels=['O'])
         self.csv_output = csv_output
 
@@ -86,11 +86,14 @@ class DTraceBenchmarkDriver(perf_test_driver.BenchmarkDriver):
         sys.stdout.flush()
 
         def get_results_with_iters(iters):
+            e = os.environ
+            e['SWIFT_DETERMINISTIC_HASHING'] = '1'
             p = subprocess.Popen([
                 'sudo', 'dtrace', '-s', DTRACE_PATH,
-                '-c', '%s %s %s' % (data['path'], data['test_name'],
-                                    '--num-iters=%d' % iters)
-            ], stdout=subprocess.PIPE, stderr=open('/dev/null', 'w'))
+                '-c', '%s %s %s %s' % (data['path'], data['test_name'],
+                                       '--num-iters=%d' % iters,
+                                       '--num-samples=2')
+            ], stdout=subprocess.PIPE, stderr=open('/dev/null', 'w'), env=e)
             results = [x for x in p.communicate()[0].split("\n") if len(x) > 0]
             return [
                 x.split(',')[1] for x in

benchmark/utils/DriverUtils.swift

@@ -412,9 +412,10 @@ final class TestRunner {
   private static func getExecutedInstructions() -> UInt64 {
     if #available(OSX 10.9, iOS 7.0, *) {
       var u = rusage_info_v4()
-      let p = UnsafeMutablePointer(&u)
-      p.withMemoryRebound(to: Optional<rusage_info_t>.self, capacity: 1) { up in
-        let _ = proc_pid_rusage(getpid(), RUSAGE_INFO_V4, up)
+      withUnsafeMutablePointer(to: &u) { p in
+        p.withMemoryRebound(to: Optional<rusage_info_t>.self, capacity: 1) { up in
+          let _ = proc_pid_rusage(getpid(), RUSAGE_INFO_V4, up)
+        }
       }
       return u.ri_instructions
     } else {

cmake/modules/AddSwift.cmake

@@ -332,6 +332,8 @@ function(_add_variant_c_compile_flags)
       list(APPEND result -isystem;${path})
     endforeach()
     list(APPEND result "-D__ANDROID_API__=${SWIFT_ANDROID_API_LEVEL}")
+  elseif("${CFLAGS_SDK}" STREQUAL "WASI")
+    list(APPEND result "-D_WASI_EMULATED_MMAN")
   elseif(CFLAGS_SDK STREQUAL WINDOWS)
     swift_windows_include_for_arch(${CFLAGS_ARCH} ${CFLAGS_ARCH}_INCLUDE)
     foreach(path ${${CFLAGS_ARCH}_INCLUDE})
@@ -391,6 +393,8 @@ function(_add_variant_swift_compile_flags
     foreach(path IN LISTS ${arch}_swift_include)
       list(APPEND result "\"${CMAKE_INCLUDE_FLAG_C}${path}\"")
     endforeach()
+  elseif("${sdk}" STREQUAL "WASI")
+    list(APPEND result "-Xcc" "-D_WASI_EMULATED_MMAN")
   endif()
 
   if(NOT BUILD_STANDALONE)
@@ -500,6 +504,8 @@ function(_add_variant_link_flags)
     foreach(path IN LISTS ${LFLAGS_ARCH}_LIB)
       list(APPEND library_search_directories ${path})
     endforeach()
+  elseif("${LFLAGS_SDK}" STREQUAL "WASI")
+    list(APPEND result "-Wl,wasi-emulated-mman")
   else()
     # If lto is enabled, we need to add the object path flag so that the LTO code
     # generator leaves the intermediate object file in a place where it will not
@@ -996,7 +1002,8 @@ function(_add_swift_library_single target name)
               ${INCORPORATED_OBJECT_LIBRARIES_EXPRESSIONS}
               ${SWIFTLIB_SINGLE_XCODE_WORKAROUND_SOURCES})
   if(("${SWIFT_SDK_${SWIFTLIB_SINGLE_SDK}_OBJECT_FORMAT}" STREQUAL "ELF" OR
-      "${SWIFT_SDK_${SWIFTLIB_SINGLE_SDK}_OBJECT_FORMAT}" STREQUAL "COFF") AND
+      "${SWIFT_SDK_${SWIFTLIB_SINGLE_SDK}_OBJECT_FORMAT}" STREQUAL "COFF" OR
+      "${SWIFT_SDK_${SWIFTLIB_SINGLE_SDK}_OBJECT_FORMAT}" STREQUAL "WASM") AND
      SWIFTLIB_SINGLE_TARGET_LIBRARY)
     if("${libkind}" STREQUAL "SHARED" AND NOT SWIFTLIB_SINGLE_NOSWIFTRT)
       # TODO(compnerd) switch to the generator expression when cmake is upgraded
@@ -1315,6 +1322,12 @@ function(_add_swift_library_single target name)
       list(APPEND c_compile_flags -D_WINDLL)
     endif()
   endif()
+  # Double-check that we're not trying to build a dynamic library for WASM.
+  if(SWIFTLIB_SINGLE_SDK MATCHES WASM)
+    if(libkind STREQUAL SHARED)
+      message(FATAL_ERROR "WASM does not support shared libraries.")
+    endif()
+  endif()
   _add_variant_link_flags(
     SDK "${SWIFTLIB_SINGLE_SDK}"
     ARCH "${SWIFTLIB_SINGLE_ARCHITECTURE}"
@@ -1612,6 +1625,9 @@ endfunction()
 # SWIFT_MODULE_DEPENDS_HAIKU
 #   Swift modules this library depends on when built for Haiku.
 #
+# SWIFT_MODULE_DEPENDS_WASI
+#   Swift modules this library depends on when built for WASI.
+#
 # FRAMEWORK_DEPENDS
 #   System frameworks this library depends on.
 #
@@ -1720,7 +1736,9 @@ function(add_swift_target_library name)
         SWIFT_MODULE_DEPENDS_OSX
         SWIFT_MODULE_DEPENDS_TVOS
         SWIFT_MODULE_DEPENDS_WATCHOS
+	SWIFT_MODULE_DEPENDS_WASI
         SWIFT_MODULE_DEPENDS_WINDOWS
+        SWIFT_MODULE_DEPENDS_FROM_SDK
         TARGET_SDKS)
 
   cmake_parse_arguments(SWIFTLIB
@@ -1829,6 +1847,9 @@ function(add_swift_target_library name)
     elseif(${sdk} STREQUAL HAIKU)
       list(APPEND swiftlib_module_depends_flattened
            ${SWIFTLIB_SWIFT_MODULE_DEPENDS_HAIKU})
+    elseif(${sdk} STREQUAL WASI)
+      list(APPEND swiftlib_module_depends_flattened
+           ${SWIFTLIB_SWIFT_MODULE_DEPENDS_WASI})
     elseif(${sdk} STREQUAL WINDOWS)
       list(APPEND swiftlib_module_depends_flattened
            ${SWIFTLIB_SWIFT_MODULE_DEPENDS_WINDOWS})
@@ -1960,6 +1981,9 @@ function(add_swift_target_library name)
       set(swiftlib_c_compile_flags_all ${SWIFTLIB_C_COMPILE_FLAGS})
       if(sdk IN_LIST SWIFT_APPLE_PLATFORMS AND SWIFTLIB_IS_SDK_OVERLAY)
         set(swiftlib_swift_compile_private_frameworks_flag "-Fsystem" "${SWIFT_SDK_${sdk}_ARCH_${arch}_PATH}/System/Library/PrivateFrameworks/")
+        foreach(tbd_lib ${SWIFTLIB_SWIFT_MODULE_DEPENDS_FROM_SDK})
+          list(APPEND swiftlib_link_flags_all "${SWIFT_SDK_${sdk}_ARCH_${arch}_PATH}/usr/lib/swift/libswift${tbd_lib}.tbd")
+        endforeach()
       endif()
 
       list(APPEND swiftlib_c_compile_flags_all "-DSWIFT_TARGET_LIBRARY_NAME=${name}")