diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 26dfbd28..aa5542cc 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -16,6 +16,7 @@ jobs:
# We cache them using GitHub Actions cache and export, making the scripts below a bit more complex.
check-format:
if: github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
+ name: Check format
runs-on: ubuntu-18.04
steps:
- name: Checkout
@@ -23,7 +24,7 @@ jobs:
- name: Setup .NET Core SDK
uses: actions/setup-dotnet@v1
with:
- dotnet-version: ${{ env.dotnet_5_version }}
+ dotnet-version: ${{ env.dotnet_5_version }}
- name: Code formating check
run: |
dotnet tool restore
@@ -47,13 +48,17 @@ jobs:
strategy:
matrix:
os: [ubuntu-18.04, macos-latest, windows-latest]
+ arch: [x64, arm64]
+ exclude:
+ - os: windows-latest
+ arch: arm64
fail-fast: false
- name: Build native library (${{ matrix.os }})
+ name: Build native ${{ matrix.arch }} library (${{ matrix.os }})
runs-on: ${{ matrix.os }}
steps:
- name: Checkout
uses: actions/checkout@v2
-
+
# Find out which vcpkg version we are using.
- name: Resolve vcpkg version
id: resolve-vcpkg-version
@@ -64,22 +69,48 @@ jobs:
if [ ${#vcpkg_ref} -eq 40 ]; then commit_id=$vcpkg_ref; else commit_id=$(git ls-remote $vcpkg_url $vcpkg_ref | cut -f1); fi
echo "::set-output name=commit-id::$commit_id"
shell: bash
-
+
+ # Compute vcpkg triplet
+ - name: Compute vcpkg triplet
+ id: vcpkg-triplet
+ run: |
+ triplet="${{ matrix.arch }}-"
+ case ${{ runner.os }} in
+ Linux)
+ triplet+="linux"
+ ;;
+ macOS)
+ triplet+="osx"
+ ;;
+ Windows)
+ triplet+="windows-static"
+ ;;
+ esac
+ echo "::set-output name=triplet::$triplet"
+ shell: bash
+
# Check for cached vcpkg dependencies (use these if we can).
- name: Get cached vcpkg dependencies
id: get-cached-vcpkg
uses: actions/cache@v1
with:
path: cache/vcpkg
- key: vcpkg-${{ runner.os }}-${{ steps.resolve-vcpkg-version.outputs.commit-id }}
+ key: vcpkg-${{ steps.vcpkg-triplet.outputs.triplet }}-${{ steps.resolve-vcpkg-version.outputs.commit-id }}
- name: Use cached vcpkg dependencies
if: steps.get-cached-vcpkg.outputs.cache-hit == 'true'
run: |
mkdir build
- mv cache/vcpkg build/vcpkg.$(echo ${{ runner.os }})
+ mv cache/vcpkg build/vcpkg.${{ steps.vcpkg-triplet.outputs.triplet }}
shell: bash
-
- # Build vcpkg dependencies (if no cach hit).
+
+ # Install arm64 cross-compilation toolchain if required
+ - name: Install arm64 cross-compilation toolchain
+ if: runner.os == 'Linux' && matrix.arch == 'arm64'
+ run: |
+ sudo apt-get update
+ sudo apt-get --yes install g++-aarch64-linux-gnu pkg-config-aarch64-linux-gnu
+
+ # Build vcpkg dependencies (if no cache hit).
- name: Install vcpkg build dependencies (Linux)
if: steps.get-cached-vcpkg.outputs.cache-hit != 'true' && runner.os == 'Linux'
run: |
@@ -90,7 +121,7 @@ jobs:
run: brew install bison pkg-config
- name: Compile vcpkg dependencies (Unix)
if: steps.get-cached-vcpkg.outputs.cache-hit != 'true' && (runner.os == 'Linux' || runner.os == 'macOS')
- run: ./vcpkg_unix.sh
+ run: ./vcpkg_unix.sh ${{ matrix.arch }}
- name: Compile vcpkg dependencies (Windows)
if: steps.get-cached-vcpkg.outputs.cache-hit != 'true' && runner.os == 'Windows'
run: ./vcpkg_windows.bat
@@ -98,17 +129,17 @@ jobs:
if: steps.get-cached-vcpkg.outputs.cache-hit != 'true' && (success() || failure())
uses: actions/upload-artifact@v2
with:
- name: ${{ runner.os }}-vcpkg-arrow-logs
- path: build/vcpkg.${{ runner.os }}/buildtrees/arrow/*.log
+ name: ${{ steps.vcpkg-triplet.outputs.triplet }}-vcpkg-arrow-logs
+ path: build/vcpkg.${{ steps.vcpkg-triplet.outputs.triplet }}/buildtrees/arrow/*.log
- name: Cleanup vcpkg build
if: steps.get-cached-vcpkg.outputs.cache-hit != 'true'
- run: rm -rf build/vcpkg.$(echo ${{ runner.os }} | tr A-Z a-z)/{buildtrees,downloads}
+ run: rm -rf build/vcpkg.${{ steps.vcpkg-triplet.outputs.triplet }}/{buildtrees,downloads}
shell: bash
- name: Export vcpkg dependencies
if: steps.get-cached-vcpkg.outputs.cache-hit != 'true'
- run: build/vcpkg.$(echo ${{ runner.os }})/vcpkg export --x-all-installed --raw --output=../../cache/vcpkg
+ run: build/vcpkg.${{ steps.vcpkg-triplet.outputs.triplet }}/vcpkg export --x-all-installed --raw --output=../../cache/vcpkg
shell: bash
-
+
# .NET Core Setup (and also MSBuild for Windows).
- name: Setup .NET Core
uses: actions/setup-dotnet@v1
@@ -123,48 +154,41 @@ jobs:
- name: Setup MSBuild
if: runner.os == 'Windows'
uses: microsoft/setup-msbuild@v1
-
+
# Compile ParquetSharp (and upload the native library as an artifact).
- name: Compile native ParquetSharp library (Unix)
if: runner.os == 'Linux' || runner.os == 'macOS'
- run: ./build_unix.sh
+ run: ./build_unix.sh ${{ matrix.arch }}
- name: Compile native ParquetSharp library (Windows)
if: runner.os == 'Windows'
run: ./build_windows.bat
- name: Build .NET benchmarks & unit tests
run: |
- dotnet build csharp.benchmark --configuration=Release
- dotnet build csharp.test --configuration=Release
+ dotnet build csharp.benchmark --configuration=Release -p:OSArchitecture=${{ matrix.arch }}
+ dotnet build csharp.test --configuration=Release -p:OSArchitecture=${{ matrix.arch }}
- name: Upload native ParquetSharp library
uses: actions/upload-artifact@v2
with:
- name: ${{ runner.os }}-native-library
+ name: ${{ steps.vcpkg-triplet.outputs.triplet }}-native-library
path: bin
-
+
# Download all native shared libraries and create the nuget package.
# Upload nuget package as an artifact.
build-nuget:
name: Build NuGet package
- runs-on: windows-latest
+ runs-on: ubuntu-latest
needs: build-native
steps:
- name: Checkout
uses: actions/checkout@v2
- - name: Download native ParquetSharp library (Linux)
- uses: actions/download-artifact@v2
- with:
- name: Linux-native-library
- path: bin
- - name: Download native ParquetSharp library (Windows)
- uses: actions/download-artifact@v2
- with:
- name: Windows-native-library
- path: bin
- - name: Download native ParquetSharp library (macOS)
+ - name: Download all artifacts
uses: actions/download-artifact@v2
with:
- name: macOS-native-library
- path: bin
+ path: artifacts
+ - name: Copy native ParquetSharp libraries
+ run: |
+ mkdir bin
+ cp -rv artifacts/*-native-library/* bin/
- name: Setup .NET Core
uses: actions/setup-dotnet@v1
with:
@@ -173,7 +197,7 @@ jobs:
run: dotnet new globaljson --sdk-version ${{ env.dotnet_5_version }}
- name: Build NuGet package
run: dotnet build csharp --configuration=Release
- - name: Upload NuGet artifact
+ - name: Upload NuGet artifact
uses: actions/upload-artifact@v2
with:
name: nuget-package
@@ -185,11 +209,16 @@ jobs:
matrix:
os: [ubuntu-18.04, ubuntu-20.04, macos-latest, windows-latest]
dotnet: [3, 5]
+ arch: [x64]
include:
- os: windows-latest
dotnet: 4
+ arch: x64
+ - os: ubuntu-20.04
+ dotnet: 5
+ arch: arm64
fail-fast: false
- name: Test NuGet package (.NET ${{ matrix.dotnet }} on ${{ matrix.os }})
+ name: Test NuGet package (.NET ${{ matrix.dotnet }} ${{ matrix.arch }} on ${{ matrix.os }})
runs-on: ${{ matrix.os }}
needs: build-nuget
steps:
@@ -204,16 +233,6 @@ jobs:
with:
name: nuget-package
path: nuget
- - name: Create local NuGet feed (Unix)
- if: runner.os == 'Linux' || runner.os == 'macOS'
- run: |
- curl -sLO https://dist.nuget.org/win-x86-commandline/v5.5.1/nuget.exe
- mono nuget.exe add -source local nuget/ParquetSharp.${{ steps.get-version.outputs.version }}.nupkg
- - name: Create local NuGet feed (Windows)
- if: runner.os == 'Windows'
- run: |
- choco install nuget.commandline
- nuget add -source local nuget/ParquetSharp.${{ steps.get-version.outputs.version }}.nupkg
- name: Setup .NET variables
id: dotnet
shell: bash
@@ -242,16 +261,28 @@ jobs:
- name: Pin .NET Core SDK
run: dotnet new globaljson --sdk-version ${{ steps.dotnet.outputs.version }}
- name: Add local NuGet feed
- run: dotnet nuget add source -n local $PWD/local
+ run: |
+ dotnet new nugetconfig
+ dotnet nuget add source -n local $PWD/nuget
- name: Change test project references to use local NuGet package
run: |
dotnet remove csharp.test reference csharp/ParquetSharp.csproj
dotnet add csharp.test package ParquetSharp -v ${{ steps.get-version.outputs.version }}
- - name: Build & Run .NET unit tests
+ - name: Setup QEMU for arm64
+ if: matrix.arch == 'arm64'
+ uses: docker/setup-qemu-action@v1
+ with:
+ platforms: arm64
+ - name: Build & Run .NET unit tests (x64)
+ if: matrix.arch == 'x64'
run: dotnet test csharp.test --configuration=Release --framework ${{ steps.dotnet.outputs.framework }}
+ - name: Build & Run .NET unit tests (arm64)
+ if: matrix.arch == 'arm64'
+ run: docker run --rm --platform linux/arm64/v8 -v $PWD:$PWD -w $PWD mcr.microsoft.com/dotnet/sdk:${{ steps.dotnet.outputs.version }} dotnet test csharp.test --configuration=Release --framework ${{ steps.dotnet.outputs.framework }}
# Virtual job that can be configured as a required check before a PR can be merged.
all-required-checks-done:
+ name: All required checks done
needs:
- check-format
- test-nuget
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8a5c9032..5cde8f43 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -12,15 +12,15 @@ if (UNIX AND NOT CMAKE_BUILD_TYPE)
endif ()
set(CMAKE_DEBUG_POSTFIX d)
-set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin)
-set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin)
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin/${VCPKG_TARGET_TRIPLET})
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin/${VCPKG_TARGET_TRIPLET})
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/")
set (CMAKE_CXX_STANDARD 17)
foreach (OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES})
string(TOUPPER ${OUTPUTCONFIG} OUTPUTCONFIG)
- set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${PROJECT_SOURCE_DIR}/bin)
- set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${PROJECT_SOURCE_DIR}/bin)
+ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${PROJECT_SOURCE_DIR}/bin/${VCPKG_TARGET_TRIPLET})
+ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${PROJECT_SOURCE_DIR}/bin/${VCPKG_TARGET_TRIPLET})
endforeach()
if (MSVC)
diff --git a/build_unix.sh b/build_unix.sh
index 35674a8f..30a8a0d7 100755
--- a/build_unix.sh
+++ b/build_unix.sh
@@ -1,14 +1,34 @@
#!/bin/sh
set -e
+case ${1:-$(uname -m)} in
+ x86_64|x64)
+ vcpkg_arch=x64
+ linux_arch=x86_64
+ osx_arch=x86_64
+ ;;
+ aarch64|arm64)
+ vcpkg_arch=arm64
+ linux_arch=aarch64
+ osx_arch=arm64
+ ;;
+ *)
+ echo "Architecture not supported"
+ exit 1
+ ;;
+esac
+
case $(uname) in
Linux)
- os=Linux
- triplet=x64-linux
+ os=linux
+ options="-D CMAKE_SYSTEM_PROCESSOR=$linux_arch \
+ -D CMAKE_C_COMPILER=$(which $linux_arch-linux-gnu-gcc) \
+ -D CMAKE_CXX_COMPILER=$(which $linux_arch-linux-gnu-g++) \
+ -D CMAKE_STRIP=$(which $linux_arch-linux-gnu-strip)"
;;
Darwin)
- os=macOS
- triplet=x64-osx
+ os=osx
+ options="-D CMAKE_OSX_ARCHITECTURES=$osx_arch"
;;
*)
echo "OS not supported"
@@ -16,7 +36,7 @@ case $(uname) in
;;
esac
-mkdir -p build/$os
-cd build/$os
-cmake -D VCPKG_TARGET_TRIPLET=$triplet -D CMAKE_TOOLCHAIN_FILE=../vcpkg.$os/scripts/buildsystems/vcpkg.cmake ../..
-make -j
+triplet=$vcpkg_arch-$os
+
+cmake -B build/$triplet -S . -D VCPKG_TARGET_TRIPLET=$triplet -D CMAKE_TOOLCHAIN_FILE=../vcpkg.$triplet/scripts/buildsystems/vcpkg.cmake $options
+cmake --build build/$triplet -j
diff --git a/build_windows.bat b/build_windows.bat
index c5c8bcef..ea532165 100644
--- a/build_windows.bat
+++ b/build_windows.bat
@@ -1,10 +1,6 @@
-cd build || goto :error
-mkdir Windows || goto :error
-cd Windows || goto :error
-cmake -D VCPKG_TARGET_TRIPLET=x64-windows-static -D CMAKE_TOOLCHAIN_FILE=../vcpkg.Windows/scripts/buildsystems/vcpkg.cmake -G "Visual Studio 16 2019" -A "x64" ../.. || goto :error
-msbuild ParquetSharp.sln -t:ParquetSharpNative:Rebuild -p:Configuration=Release || goto :error
-cd ..
-cd ..
+set triplet=x64-windows-static
+cmake -B build/%triplet% -S . -D VCPKG_TARGET_TRIPLET=%triplet% -D CMAKE_TOOLCHAIN_FILE=../vcpkg.%triplet%/scripts/buildsystems/vcpkg.cmake -G "Visual Studio 16 2019" -A "x64" || goto :error
+msbuild build/%triplet%/ParquetSharp.sln -t:ParquetSharpNative:Rebuild -p:Configuration=Release || goto :error
exit /b
diff --git a/csharp.benchmark/ParquetSharp.Benchmark.csproj b/csharp.benchmark/ParquetSharp.Benchmark.csproj
index d25ada6f..371062f1 100644
--- a/csharp.benchmark/ParquetSharp.Benchmark.csproj
+++ b/csharp.benchmark/ParquetSharp.Benchmark.csproj
@@ -7,8 +7,6 @@
enable
ParquetSharp.Benchmark
ParquetSharp.Benchmark
- x64
- x64
true
diff --git a/csharp.test/ParquetSharp.Test.csproj b/csharp.test/ParquetSharp.Test.csproj
index 34392335..11fa5658 100644
--- a/csharp.test/ParquetSharp.Test.csproj
+++ b/csharp.test/ParquetSharp.Test.csproj
@@ -7,8 +7,7 @@
enable
ParquetSharp.Test
ParquetSharp.Test
- x64
- x64
+ x64
false
true
true
diff --git a/csharp/ParquetSharp.csproj b/csharp/ParquetSharp.csproj
index 34b14efa..3c3ae73a 100644
--- a/csharp/ParquetSharp.csproj
+++ b/csharp/ParquetSharp.csproj
@@ -6,8 +6,7 @@
enable
ParquetSharp
ParquetSharp
- x64
- x64
+ x64
true
true
true
@@ -40,28 +39,41 @@
true
true
true
+ $([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)
-
+
PreserveNewest
-
+
PreserveNewest
-
+
PreserveNewest
-
+
PreserveNewest
-
+
PreserveNewest
-
+
PreserveNewest
-
+
+ PreserveNewest
+
+
+ PreserveNewest
+
+
+ PreserveNewest
+
+
+ PreserveNewest
+
+
PreserveNewest
diff --git a/vcpkg_unix.sh b/vcpkg_unix.sh
index e3493776..d0e432ed 100755
--- a/vcpkg_unix.sh
+++ b/vcpkg_unix.sh
@@ -1,14 +1,25 @@
#!/bin/bash
set -e
+case ${1:-$(uname -m)} in
+ x86_64|x64)
+ arch=x64
+ ;;
+ aarch64|arm64)
+ arch=arm64
+ ;;
+ *)
+ echo "Architecture not supported"
+ exit 1
+ ;;
+esac
+
case $(uname) in
Linux)
- os=Linux
- triplet=x64-linux
+ os=linux
;;
Darwin)
- os=macOS
- triplet=x64-osx
+ os=osx
if ! which brew >/dev/null || [ ! -x $(brew --prefix)/opt/bison/bin/bison ]
then
echo 'The version of bison provided with macOS is too old.'
@@ -25,6 +36,8 @@ case $(uname) in
;;
esac
+triplet=$arch-$os
+
# Make sure reading vcpkg_version.txt works even when it doesn't end with a newline
read -r vcpkg_url vcpkg_ref << EOF
$(cat vcpkg_version.txt)
@@ -33,9 +46,19 @@ EOF
mkdir -p build
cd build
# Clone without checking out a branch, as vcpkg_ref could be a commit SHA
-git clone $vcpkg_url vcpkg.$os
-cd vcpkg.$os
+git clone $vcpkg_url vcpkg.$triplet
+cd vcpkg.$triplet
git checkout $vcpkg_ref
+
+# Only build release configuration in CI
+if [ "$GITHUB_ACTIONS" == "true" ]
+then
+ for triplet_file in triplets/{,community/}$triplet.cmake
+ do
+ [ -f "$triplet_file" ] && echo "set(VCPKG_BUILD_TYPE release)" >> "$triplet_file"
+ done
+fi
+
./bootstrap-vcpkg.sh
./vcpkg install arrow:$triplet
diff --git a/vcpkg_version.txt b/vcpkg_version.txt
index c77c82bb..2ee01113 100644
--- a/vcpkg_version.txt
+++ b/vcpkg_version.txt
@@ -1 +1 @@
-https://github.com/microsoft/vcpkg.git ba69c09fab296d975ee4f578d070f9c67da21dd5
+https://github.com/microsoft/vcpkg.git 77c4e0cb087b3cecb5525dacd1fad18a0bb23573
diff --git a/vcpkg_windows.bat b/vcpkg_windows.bat
index e0343cbd..de7c5927 100644
--- a/vcpkg_windows.bat
+++ b/vcpkg_windows.bat
@@ -3,16 +3,18 @@ for /f "tokens=1,2" %%a in (vcpkg_version.txt) do (
set vcpkg_ref=%%b
) || goto :error
+set triplet=x64-windows-static
+
mkdir build
cd build || goto :error
rem Clone without checking out a branch, as vcpkg_ref could be a commit SHA
-git clone %vcpkg_url% vcpkg.Windows || goto :error
-cd vcpkg.Windows || goto :error
+git clone %vcpkg_url% vcpkg.%triplet% || goto :error
+cd vcpkg.%triplet% || goto :error
git checkout %vcpkg_ref% || goto :error
-if "%GITHUB_ACTIONS%"=="true" echo set(VCPKG_BUILD_TYPE release) >> triplets\x64-windows-static.cmake || goto :error
+if "%GITHUB_ACTIONS%"=="true" echo set(VCPKG_BUILD_TYPE release) >> triplets\%triplet%.cmake || goto :error
call .\bootstrap-vcpkg.bat || goto :error
-.\vcpkg.exe install arrow:x64-windows-static || goto :error
+.\vcpkg.exe install arrow:%triplet% || goto :error
cd ..
cd ..