Enable support for arm64

.github/workflows/ci.yml

@@ -16,14 +16,15 @@ jobs:
   # We cache them using GitHub Actions cache and export, making the scripts below a bit more complex.
   check-format:
     if: github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
+    name: Check format
     runs-on: ubuntu-18.04
     steps:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Setup .NET Core SDK
         uses: actions/setup-dotnet@v1
         with:
-          dotnet-version: ${{ env.dotnet_5_version }} 
+          dotnet-version: ${{ env.dotnet_5_version }}
       - name: Code formating check
         run: |
           dotnet tool restore
@@ -47,13 +48,17 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-18.04, macos-latest, windows-latest]
+        arch: [x64, arm64]
+        exclude:
+        - os: windows-latest
+          arch: arm64
       fail-fast: false
-    name: Build native library (${{ matrix.os }})
+    name: Build native ${{ matrix.arch }} library (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     steps:
     - name: Checkout
       uses: actions/checkout@v2
-      
+
     # Find out which vcpkg version we are using.
     - name: Resolve vcpkg version
       id: resolve-vcpkg-version
@@ -64,22 +69,48 @@ jobs:
         if [ ${#vcpkg_ref} -eq 40 ]; then commit_id=$vcpkg_ref; else commit_id=$(git ls-remote $vcpkg_url $vcpkg_ref | cut -f1); fi
         echo "::set-output name=commit-id::$commit_id"
       shell: bash
-
+
+    # Compute vcpkg triplet
+    - name: Compute vcpkg triplet
+      id: vcpkg-triplet
+      run: |
+        triplet="${{ matrix.arch }}-"
+        case ${{ runner.os }} in
+          Linux)
+            triplet+="linux"
+            ;;
+          macOS)
+            triplet+="osx"
+            ;;
+          Windows)
+            triplet+="windows-static"
+            ;;
+        esac
+        echo "::set-output name=triplet::$triplet"
+      shell: bash
+
     # Check for cached vcpkg dependencies (use these if we can).
     - name: Get cached vcpkg dependencies
       id: get-cached-vcpkg
       uses: actions/cache@v1
       with:
         path: cache/vcpkg
-        key: vcpkg-${{ runner.os }}-${{ steps.resolve-vcpkg-version.outputs.commit-id }}
+        key: vcpkg-${{ steps.vcpkg-triplet.outputs.triplet }}-${{ steps.resolve-vcpkg-version.outputs.commit-id }}
     - name: Use cached vcpkg dependencies
       if: steps.get-cached-vcpkg.outputs.cache-hit == 'true'
       run: |
         mkdir build
-        mv cache/vcpkg build/vcpkg.$(echo ${{ runner.os }})
+        mv cache/vcpkg build/vcpkg.${{ steps.vcpkg-triplet.outputs.triplet }}
       shell: bash
-
-    # Build vcpkg dependencies (if no cach hit).
+
+    # Install arm64 cross-compilation toolchain if required
+    - name: Install arm64 cross-compilation toolchain
+      if: runner.os == 'Linux' && matrix.arch == 'arm64'
+      run: |
+        sudo apt-get update
+        sudo apt-get --yes install g++-aarch64-linux-gnu pkg-config-aarch64-linux-gnu
+
+    # Build vcpkg dependencies (if no cache hit).
     - name: Install vcpkg build dependencies (Linux)
       if: steps.get-cached-vcpkg.outputs.cache-hit != 'true' && runner.os == 'Linux'
       run: |
@@ -90,25 +121,25 @@ jobs:
       run: brew install bison pkg-config
     - name: Compile vcpkg dependencies (Unix)
       if: steps.get-cached-vcpkg.outputs.cache-hit != 'true' && (runner.os == 'Linux' || runner.os == 'macOS')
-      run: ./vcpkg_unix.sh
+      run: ./vcpkg_unix.sh ${{ matrix.arch }}
     - name: Compile vcpkg dependencies (Windows)
       if: steps.get-cached-vcpkg.outputs.cache-hit != 'true' && runner.os == 'Windows'
       run: ./vcpkg_windows.bat
     - name: Upload vcpkg arrow logs
       if: steps.get-cached-vcpkg.outputs.cache-hit != 'true' && (success() || failure())
       uses: actions/upload-artifact@v2
       with:
-        name: ${{ runner.os }}-vcpkg-arrow-logs
-        path: build/vcpkg.${{ runner.os }}/buildtrees/arrow/*.log
+        name: ${{ steps.vcpkg-triplet.outputs.triplet }}-vcpkg-arrow-logs
+        path: build/vcpkg.${{ steps.vcpkg-triplet.outputs.triplet }}/buildtrees/arrow/*.log
     - name: Cleanup vcpkg build
       if: steps.get-cached-vcpkg.outputs.cache-hit != 'true'
-      run: rm -rf build/vcpkg.$(echo ${{ runner.os }} | tr A-Z a-z)/{buildtrees,downloads}
+      run: rm -rf build/vcpkg.${{ steps.vcpkg-triplet.outputs.triplet }}/{buildtrees,downloads}
       shell: bash
     - name: Export vcpkg dependencies
       if: steps.get-cached-vcpkg.outputs.cache-hit != 'true'
-      run: build/vcpkg.$(echo ${{ runner.os }})/vcpkg export --x-all-installed --raw --output=../../cache/vcpkg
+      run: build/vcpkg.${{ steps.vcpkg-triplet.outputs.triplet }}/vcpkg export --x-all-installed --raw --output=../../cache/vcpkg
       shell: bash
-      
+
     # .NET Core Setup (and also MSBuild for Windows).
     - name: Setup .NET Core
       uses: actions/setup-dotnet@v1
@@ -123,48 +154,41 @@ jobs:
     - name: Setup MSBuild
       if: runner.os == 'Windows'
       uses: microsoft/setup-msbuild@v1
-      
+
     # Compile ParquetSharp (and upload the native library as an artifact).
     - name: Compile native ParquetSharp library (Unix)
       if: runner.os == 'Linux' || runner.os == 'macOS'
-      run: ./build_unix.sh
+      run: ./build_unix.sh ${{ matrix.arch }}
     - name: Compile native ParquetSharp library (Windows)
       if: runner.os == 'Windows'
       run: ./build_windows.bat
     - name: Build .NET benchmarks & unit tests
       run: |
-        dotnet build csharp.benchmark --configuration=Release
-        dotnet build csharp.test --configuration=Release
+        dotnet build csharp.benchmark --configuration=Release -p:OSArchitecture=${{ matrix.arch }}
+        dotnet build csharp.test --configuration=Release -p:OSArchitecture=${{ matrix.arch }}
     - name: Upload native ParquetSharp library
       uses: actions/upload-artifact@v2
       with:
-        name: ${{ runner.os }}-native-library
+        name: ${{ steps.vcpkg-triplet.outputs.triplet }}-native-library
         path: bin
-        
+
   # Download all native shared libraries and create the nuget package.
   # Upload nuget package as an artifact.
   build-nuget:
     name: Build NuGet package
-    runs-on: windows-latest    
+    runs-on: ubuntu-latest
     needs: build-native
     steps:
     - name: Checkout
       uses: actions/checkout@v2
-    - name: Download native ParquetSharp library (Linux)
-      uses: actions/download-artifact@v2
-      with:
-        name: Linux-native-library
-        path: bin
-    - name: Download native ParquetSharp library (Windows)
-      uses: actions/download-artifact@v2
-      with:
-        name: Windows-native-library
-        path: bin
-    - name: Download native ParquetSharp library (macOS)
+    - name: Download all artifacts
       uses: actions/download-artifact@v2
       with:
-        name: macOS-native-library
-        path: bin
+        path: artifacts
+    - name: Copy native ParquetSharp libraries
+      run: |
+        mkdir bin
+        cp -rv artifacts/*-native-library/* bin/
     - name: Setup .NET Core
       uses: actions/setup-dotnet@v1
       with:
@@ -173,7 +197,7 @@ jobs:
       run: dotnet new globaljson --sdk-version ${{ env.dotnet_5_version }}
     - name: Build NuGet package
       run: dotnet build csharp --configuration=Release
-    - name: Upload NuGet artifact 
+    - name: Upload NuGet artifact
       uses: actions/upload-artifact@v2
       with:
         name: nuget-package
@@ -185,11 +209,16 @@ jobs:
       matrix:
         os: [ubuntu-18.04, ubuntu-20.04, macos-latest, windows-latest]
         dotnet: [3, 5]
+        arch: [x64]
         include:
         - os: windows-latest
           dotnet: 4
+          arch: x64
+        - os: ubuntu-20.04
+          dotnet: 5
+          arch: arm64
       fail-fast: false
-    name: Test NuGet package (.NET ${{ matrix.dotnet }} on ${{ matrix.os }})
+    name: Test NuGet package (.NET ${{ matrix.dotnet }} ${{ matrix.arch }} on ${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     needs: build-nuget
     steps:
@@ -204,16 +233,6 @@ jobs:
       with:
         name: nuget-package
         path: nuget
-    - name: Create local NuGet feed (Unix)
-      if: runner.os == 'Linux' || runner.os == 'macOS'
-      run: |
-        curl -sLO https://dist.nuget.org/win-x86-commandline/v5.5.1/nuget.exe
-        mono nuget.exe add -source local nuget/ParquetSharp.${{ steps.get-version.outputs.version }}.nupkg
-    - name: Create local NuGet feed (Windows)
-      if: runner.os == 'Windows'
-      run: |
-        choco install nuget.commandline
-        nuget add -source local nuget/ParquetSharp.${{ steps.get-version.outputs.version }}.nupkg
     - name: Setup .NET variables
       id: dotnet
       shell: bash
@@ -242,16 +261,28 @@ jobs:
     - name: Pin .NET Core SDK
       run: dotnet new globaljson --sdk-version ${{ steps.dotnet.outputs.version }}
     - name: Add local NuGet feed
-      run: dotnet nuget add source -n local $PWD/local
+      run: |
+        dotnet new nugetconfig
+        dotnet nuget add source -n local $PWD/nuget
     - name: Change test project references to use local NuGet package
       run: |
         dotnet remove csharp.test reference csharp/ParquetSharp.csproj
         dotnet add csharp.test package ParquetSharp -v ${{ steps.get-version.outputs.version }}
-    - name: Build & Run .NET unit tests
+    - name: Setup QEMU for arm64
+      if: matrix.arch == 'arm64'
+      uses: docker/setup-qemu-action@v1
+      with:
+        platforms: arm64
+    - name: Build & Run .NET unit tests (x64)
+      if: matrix.arch == 'x64'
       run: dotnet test csharp.test --configuration=Release --framework ${{ steps.dotnet.outputs.framework }}
+    - name: Build & Run .NET unit tests (arm64)
+      if: matrix.arch == 'arm64'
+      run: docker run --rm --platform linux/arm64/v8 -v $PWD:$PWD -w $PWD mcr.microsoft.com/dotnet/sdk:${{ steps.dotnet.outputs.version }} dotnet test csharp.test --configuration=Release --framework ${{ steps.dotnet.outputs.framework }}
 
   # Virtual job that can be configured as a required check before a PR can be merged.
   all-required-checks-done:
+    name: All required checks done
     needs:
       - check-format
       - test-nuget

CMakeLists.txt

@@ -12,15 +12,15 @@ if (UNIX AND NOT CMAKE_BUILD_TYPE)
 endif ()
 
 set(CMAKE_DEBUG_POSTFIX d)
-set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin)
-set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin)
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin/${VCPKG_TARGET_TRIPLET})
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin/${VCPKG_TARGET_TRIPLET})
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/")
 set (CMAKE_CXX_STANDARD 17)
 
 foreach (OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES})
     string(TOUPPER ${OUTPUTCONFIG} OUTPUTCONFIG)
-    set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${PROJECT_SOURCE_DIR}/bin)
-    set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${PROJECT_SOURCE_DIR}/bin)
+    set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${PROJECT_SOURCE_DIR}/bin/${VCPKG_TARGET_TRIPLET})
+    set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${PROJECT_SOURCE_DIR}/bin/${VCPKG_TARGET_TRIPLET})
 endforeach()
 
 if (MSVC)

build_unix.sh

@@ -1,22 +1,42 @@
 #!/bin/sh
 set -e
 
+case ${1:-$(uname -m)} in
+  x86_64|x64)
+    vcpkg_arch=x64
+    linux_arch=x86_64
+    osx_arch=x86_64
+    ;;
+  aarch64|arm64)
+    vcpkg_arch=arm64
+    linux_arch=aarch64
+    osx_arch=arm64
+    ;;
+  *)
+    echo "Architecture not supported"
+    exit 1
+    ;;
+esac
+
 case $(uname) in
   Linux)
-    os=Linux
-    triplet=x64-linux
+    os=linux
+    options="-D CMAKE_SYSTEM_PROCESSOR=$linux_arch \
+             -D CMAKE_C_COMPILER=$(which $linux_arch-linux-gnu-gcc) \
+             -D CMAKE_CXX_COMPILER=$(which $linux_arch-linux-gnu-g++) \
+             -D CMAKE_STRIP=$(which $linux_arch-linux-gnu-strip)"
     ;;
   Darwin)
-    os=macOS
-    triplet=x64-osx
+    os=osx
+    options="-D CMAKE_OSX_ARCHITECTURES=$osx_arch"
     ;;
   *)
     echo "OS not supported"
     exit 1
     ;;
 esac
 
-mkdir -p build/$os
-cd build/$os
-cmake -D VCPKG_TARGET_TRIPLET=$triplet -D CMAKE_TOOLCHAIN_FILE=../vcpkg.$os/scripts/buildsystems/vcpkg.cmake ../..
-make -j
+triplet=$vcpkg_arch-$os
+
+cmake -B build/$triplet -S . -D VCPKG_TARGET_TRIPLET=$triplet -D CMAKE_TOOLCHAIN_FILE=../vcpkg.$triplet/scripts/buildsystems/vcpkg.cmake $options
+cmake --build build/$triplet -j

build_windows.bat

@@ -1,10 +1,6 @@
-cd build || goto :error
-mkdir Windows || goto :error
-cd Windows || goto :error
-cmake -D VCPKG_TARGET_TRIPLET=x64-windows-static -D CMAKE_TOOLCHAIN_FILE=../vcpkg.Windows/scripts/buildsystems/vcpkg.cmake -G "Visual Studio 16 2019" -A "x64" ../.. || goto :error
-msbuild ParquetSharp.sln -t:ParquetSharpNative:Rebuild -p:Configuration=Release || goto :error
-cd ..
-cd ..
+set triplet=x64-windows-static
+cmake -B build/%triplet% -S . -D VCPKG_TARGET_TRIPLET=%triplet% -D CMAKE_TOOLCHAIN_FILE=../vcpkg.%triplet%/scripts/buildsystems/vcpkg.cmake -G "Visual Studio 16 2019" -A "x64" || goto :error
+msbuild build/%triplet%/ParquetSharp.sln -t:ParquetSharpNative:Rebuild -p:Configuration=Release || goto :error
 
 exit /b
 

csharp.benchmark/ParquetSharp.Benchmark.csproj

@@ -7,8 +7,6 @@
     <Nullable>enable</Nullable>
     <AssemblyName>ParquetSharp.Benchmark</AssemblyName>
     <RootNamespace>ParquetSharp.Benchmark</RootNamespace>
-    <Platforms>x64</Platforms>
-    <PlatformTarget>x64</PlatformTarget>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
   </PropertyGroup>
 

csharp.test/ParquetSharp.Test.csproj

@@ -7,8 +7,7 @@
     <Nullable>enable</Nullable>
     <AssemblyName>ParquetSharp.Test</AssemblyName>
     <RootNamespace>ParquetSharp.Test</RootNamespace>
-    <Platforms>x64</Platforms>
-    <PlatformTarget>x64</PlatformTarget>
+    <PlatformTarget Condition="'$(TargetFramework)'=='net472'">x64</PlatformTarget>
     <GenerateProgramFile>false</GenerateProgramFile>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>