diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26dfbd28..aa5542cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,7 @@ jobs: # We cache them using GitHub Actions cache and export, making the scripts below a bit more complex. check-format: if: github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id + name: Check format runs-on: ubuntu-18.04 steps: - name: Checkout @@ -23,7 +24,7 @@ jobs: - name: Setup .NET Core SDK uses: actions/setup-dotnet@v1 with: - dotnet-version: ${{ env.dotnet_5_version }} + dotnet-version: ${{ env.dotnet_5_version }} - name: Code formating check run: | dotnet tool restore @@ -47,13 +48,17 @@ jobs: strategy: matrix: os: [ubuntu-18.04, macos-latest, windows-latest] + arch: [x64, arm64] + exclude: + - os: windows-latest + arch: arm64 fail-fast: false - name: Build native library (${{ matrix.os }}) + name: Build native ${{ matrix.arch }} library (${{ matrix.os }}) runs-on: ${{ matrix.os }} steps: - name: Checkout uses: actions/checkout@v2 - + # Find out which vcpkg version we are using. - name: Resolve vcpkg version id: resolve-vcpkg-version @@ -64,22 +69,48 @@ jobs: if [ ${#vcpkg_ref} -eq 40 ]; then commit_id=$vcpkg_ref; else commit_id=$(git ls-remote $vcpkg_url $vcpkg_ref | cut -f1); fi echo "::set-output name=commit-id::$commit_id" shell: bash - + + # Compute vcpkg triplet + - name: Compute vcpkg triplet + id: vcpkg-triplet + run: | + triplet="${{ matrix.arch }}-" + case ${{ runner.os }} in + Linux) + triplet+="linux" + ;; + macOS) + triplet+="osx" + ;; + Windows) + triplet+="windows-static" + ;; + esac + echo "::set-output name=triplet::$triplet" + shell: bash + # Check for cached vcpkg dependencies (use these if we can). - name: Get cached vcpkg dependencies id: get-cached-vcpkg uses: actions/cache@v1 with: path: cache/vcpkg - key: vcpkg-${{ runner.os }}-${{ steps.resolve-vcpkg-version.outputs.commit-id }} + key: vcpkg-${{ steps.vcpkg-triplet.outputs.triplet }}-${{ steps.resolve-vcpkg-version.outputs.commit-id }} - name: Use cached vcpkg dependencies if: steps.get-cached-vcpkg.outputs.cache-hit == 'true' run: | mkdir build - mv cache/vcpkg build/vcpkg.$(echo ${{ runner.os }}) + mv cache/vcpkg build/vcpkg.${{ steps.vcpkg-triplet.outputs.triplet }} shell: bash - - # Build vcpkg dependencies (if no cach hit). + + # Install arm64 cross-compilation toolchain if required + - name: Install arm64 cross-compilation toolchain + if: runner.os == 'Linux' && matrix.arch == 'arm64' + run: | + sudo apt-get update + sudo apt-get --yes install g++-aarch64-linux-gnu pkg-config-aarch64-linux-gnu + + # Build vcpkg dependencies (if no cache hit). - name: Install vcpkg build dependencies (Linux) if: steps.get-cached-vcpkg.outputs.cache-hit != 'true' && runner.os == 'Linux' run: | @@ -90,7 +121,7 @@ jobs: run: brew install bison pkg-config - name: Compile vcpkg dependencies (Unix) if: steps.get-cached-vcpkg.outputs.cache-hit != 'true' && (runner.os == 'Linux' || runner.os == 'macOS') - run: ./vcpkg_unix.sh + run: ./vcpkg_unix.sh ${{ matrix.arch }} - name: Compile vcpkg dependencies (Windows) if: steps.get-cached-vcpkg.outputs.cache-hit != 'true' && runner.os == 'Windows' run: ./vcpkg_windows.bat @@ -98,17 +129,17 @@ jobs: if: steps.get-cached-vcpkg.outputs.cache-hit != 'true' && (success() || failure()) uses: actions/upload-artifact@v2 with: - name: ${{ runner.os }}-vcpkg-arrow-logs - path: build/vcpkg.${{ runner.os }}/buildtrees/arrow/*.log + name: ${{ steps.vcpkg-triplet.outputs.triplet }}-vcpkg-arrow-logs + path: build/vcpkg.${{ steps.vcpkg-triplet.outputs.triplet }}/buildtrees/arrow/*.log - name: Cleanup vcpkg build if: steps.get-cached-vcpkg.outputs.cache-hit != 'true' - run: rm -rf build/vcpkg.$(echo ${{ runner.os }} | tr A-Z a-z)/{buildtrees,downloads} + run: rm -rf build/vcpkg.${{ steps.vcpkg-triplet.outputs.triplet }}/{buildtrees,downloads} shell: bash - name: Export vcpkg dependencies if: steps.get-cached-vcpkg.outputs.cache-hit != 'true' - run: build/vcpkg.$(echo ${{ runner.os }})/vcpkg export --x-all-installed --raw --output=../../cache/vcpkg + run: build/vcpkg.${{ steps.vcpkg-triplet.outputs.triplet }}/vcpkg export --x-all-installed --raw --output=../../cache/vcpkg shell: bash - + # .NET Core Setup (and also MSBuild for Windows). - name: Setup .NET Core uses: actions/setup-dotnet@v1 @@ -123,48 +154,41 @@ jobs: - name: Setup MSBuild if: runner.os == 'Windows' uses: microsoft/setup-msbuild@v1 - + # Compile ParquetSharp (and upload the native library as an artifact). - name: Compile native ParquetSharp library (Unix) if: runner.os == 'Linux' || runner.os == 'macOS' - run: ./build_unix.sh + run: ./build_unix.sh ${{ matrix.arch }} - name: Compile native ParquetSharp library (Windows) if: runner.os == 'Windows' run: ./build_windows.bat - name: Build .NET benchmarks & unit tests run: | - dotnet build csharp.benchmark --configuration=Release - dotnet build csharp.test --configuration=Release + dotnet build csharp.benchmark --configuration=Release -p:OSArchitecture=${{ matrix.arch }} + dotnet build csharp.test --configuration=Release -p:OSArchitecture=${{ matrix.arch }} - name: Upload native ParquetSharp library uses: actions/upload-artifact@v2 with: - name: ${{ runner.os }}-native-library + name: ${{ steps.vcpkg-triplet.outputs.triplet }}-native-library path: bin - + # Download all native shared libraries and create the nuget package. # Upload nuget package as an artifact. build-nuget: name: Build NuGet package - runs-on: windows-latest + runs-on: ubuntu-latest needs: build-native steps: - name: Checkout uses: actions/checkout@v2 - - name: Download native ParquetSharp library (Linux) - uses: actions/download-artifact@v2 - with: - name: Linux-native-library - path: bin - - name: Download native ParquetSharp library (Windows) - uses: actions/download-artifact@v2 - with: - name: Windows-native-library - path: bin - - name: Download native ParquetSharp library (macOS) + - name: Download all artifacts uses: actions/download-artifact@v2 with: - name: macOS-native-library - path: bin + path: artifacts + - name: Copy native ParquetSharp libraries + run: | + mkdir bin + cp -rv artifacts/*-native-library/* bin/ - name: Setup .NET Core uses: actions/setup-dotnet@v1 with: @@ -173,7 +197,7 @@ jobs: run: dotnet new globaljson --sdk-version ${{ env.dotnet_5_version }} - name: Build NuGet package run: dotnet build csharp --configuration=Release - - name: Upload NuGet artifact + - name: Upload NuGet artifact uses: actions/upload-artifact@v2 with: name: nuget-package @@ -185,11 +209,16 @@ jobs: matrix: os: [ubuntu-18.04, ubuntu-20.04, macos-latest, windows-latest] dotnet: [3, 5] + arch: [x64] include: - os: windows-latest dotnet: 4 + arch: x64 + - os: ubuntu-20.04 + dotnet: 5 + arch: arm64 fail-fast: false - name: Test NuGet package (.NET ${{ matrix.dotnet }} on ${{ matrix.os }}) + name: Test NuGet package (.NET ${{ matrix.dotnet }} ${{ matrix.arch }} on ${{ matrix.os }}) runs-on: ${{ matrix.os }} needs: build-nuget steps: @@ -204,16 +233,6 @@ jobs: with: name: nuget-package path: nuget - - name: Create local NuGet feed (Unix) - if: runner.os == 'Linux' || runner.os == 'macOS' - run: | - curl -sLO https://dist.nuget.org/win-x86-commandline/v5.5.1/nuget.exe - mono nuget.exe add -source local nuget/ParquetSharp.${{ steps.get-version.outputs.version }}.nupkg - - name: Create local NuGet feed (Windows) - if: runner.os == 'Windows' - run: | - choco install nuget.commandline - nuget add -source local nuget/ParquetSharp.${{ steps.get-version.outputs.version }}.nupkg - name: Setup .NET variables id: dotnet shell: bash @@ -242,16 +261,28 @@ jobs: - name: Pin .NET Core SDK run: dotnet new globaljson --sdk-version ${{ steps.dotnet.outputs.version }} - name: Add local NuGet feed - run: dotnet nuget add source -n local $PWD/local + run: | + dotnet new nugetconfig + dotnet nuget add source -n local $PWD/nuget - name: Change test project references to use local NuGet package run: | dotnet remove csharp.test reference csharp/ParquetSharp.csproj dotnet add csharp.test package ParquetSharp -v ${{ steps.get-version.outputs.version }} - - name: Build & Run .NET unit tests + - name: Setup QEMU for arm64 + if: matrix.arch == 'arm64' + uses: docker/setup-qemu-action@v1 + with: + platforms: arm64 + - name: Build & Run .NET unit tests (x64) + if: matrix.arch == 'x64' run: dotnet test csharp.test --configuration=Release --framework ${{ steps.dotnet.outputs.framework }} + - name: Build & Run .NET unit tests (arm64) + if: matrix.arch == 'arm64' + run: docker run --rm --platform linux/arm64/v8 -v $PWD:$PWD -w $PWD mcr.microsoft.com/dotnet/sdk:${{ steps.dotnet.outputs.version }} dotnet test csharp.test --configuration=Release --framework ${{ steps.dotnet.outputs.framework }} # Virtual job that can be configured as a required check before a PR can be merged. all-required-checks-done: + name: All required checks done needs: - check-format - test-nuget diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a5c9032..5cde8f43 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,15 +12,15 @@ if (UNIX AND NOT CMAKE_BUILD_TYPE) endif () set(CMAKE_DEBUG_POSTFIX d) -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin) -set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin/${VCPKG_TARGET_TRIPLET}) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin/${VCPKG_TARGET_TRIPLET}) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/") set (CMAKE_CXX_STANDARD 17) foreach (OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES}) string(TOUPPER ${OUTPUTCONFIG} OUTPUTCONFIG) - set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${PROJECT_SOURCE_DIR}/bin) - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${PROJECT_SOURCE_DIR}/bin) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${PROJECT_SOURCE_DIR}/bin/${VCPKG_TARGET_TRIPLET}) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${PROJECT_SOURCE_DIR}/bin/${VCPKG_TARGET_TRIPLET}) endforeach() if (MSVC) diff --git a/build_unix.sh b/build_unix.sh index 35674a8f..30a8a0d7 100755 --- a/build_unix.sh +++ b/build_unix.sh @@ -1,14 +1,34 @@ #!/bin/sh set -e +case ${1:-$(uname -m)} in + x86_64|x64) + vcpkg_arch=x64 + linux_arch=x86_64 + osx_arch=x86_64 + ;; + aarch64|arm64) + vcpkg_arch=arm64 + linux_arch=aarch64 + osx_arch=arm64 + ;; + *) + echo "Architecture not supported" + exit 1 + ;; +esac + case $(uname) in Linux) - os=Linux - triplet=x64-linux + os=linux + options="-D CMAKE_SYSTEM_PROCESSOR=$linux_arch \ + -D CMAKE_C_COMPILER=$(which $linux_arch-linux-gnu-gcc) \ + -D CMAKE_CXX_COMPILER=$(which $linux_arch-linux-gnu-g++) \ + -D CMAKE_STRIP=$(which $linux_arch-linux-gnu-strip)" ;; Darwin) - os=macOS - triplet=x64-osx + os=osx + options="-D CMAKE_OSX_ARCHITECTURES=$osx_arch" ;; *) echo "OS not supported" @@ -16,7 +36,7 @@ case $(uname) in ;; esac -mkdir -p build/$os -cd build/$os -cmake -D VCPKG_TARGET_TRIPLET=$triplet -D CMAKE_TOOLCHAIN_FILE=../vcpkg.$os/scripts/buildsystems/vcpkg.cmake ../.. -make -j +triplet=$vcpkg_arch-$os + +cmake -B build/$triplet -S . -D VCPKG_TARGET_TRIPLET=$triplet -D CMAKE_TOOLCHAIN_FILE=../vcpkg.$triplet/scripts/buildsystems/vcpkg.cmake $options +cmake --build build/$triplet -j diff --git a/build_windows.bat b/build_windows.bat index c5c8bcef..ea532165 100644 --- a/build_windows.bat +++ b/build_windows.bat @@ -1,10 +1,6 @@ -cd build || goto :error -mkdir Windows || goto :error -cd Windows || goto :error -cmake -D VCPKG_TARGET_TRIPLET=x64-windows-static -D CMAKE_TOOLCHAIN_FILE=../vcpkg.Windows/scripts/buildsystems/vcpkg.cmake -G "Visual Studio 16 2019" -A "x64" ../.. || goto :error -msbuild ParquetSharp.sln -t:ParquetSharpNative:Rebuild -p:Configuration=Release || goto :error -cd .. -cd .. +set triplet=x64-windows-static +cmake -B build/%triplet% -S . -D VCPKG_TARGET_TRIPLET=%triplet% -D CMAKE_TOOLCHAIN_FILE=../vcpkg.%triplet%/scripts/buildsystems/vcpkg.cmake -G "Visual Studio 16 2019" -A "x64" || goto :error +msbuild build/%triplet%/ParquetSharp.sln -t:ParquetSharpNative:Rebuild -p:Configuration=Release || goto :error exit /b diff --git a/csharp.benchmark/ParquetSharp.Benchmark.csproj b/csharp.benchmark/ParquetSharp.Benchmark.csproj index d25ada6f..371062f1 100644 --- a/csharp.benchmark/ParquetSharp.Benchmark.csproj +++ b/csharp.benchmark/ParquetSharp.Benchmark.csproj @@ -7,8 +7,6 @@ enable ParquetSharp.Benchmark ParquetSharp.Benchmark - x64 - x64 true diff --git a/csharp.test/ParquetSharp.Test.csproj b/csharp.test/ParquetSharp.Test.csproj index 34392335..11fa5658 100644 --- a/csharp.test/ParquetSharp.Test.csproj +++ b/csharp.test/ParquetSharp.Test.csproj @@ -7,8 +7,7 @@ enable ParquetSharp.Test ParquetSharp.Test - x64 - x64 + x64 false true true diff --git a/csharp/ParquetSharp.csproj b/csharp/ParquetSharp.csproj index 34b14efa..3c3ae73a 100644 --- a/csharp/ParquetSharp.csproj +++ b/csharp/ParquetSharp.csproj @@ -6,8 +6,7 @@ enable ParquetSharp ParquetSharp - x64 - x64 + x64 true true true @@ -40,28 +39,41 @@ true true true + $([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture) - + PreserveNewest - + PreserveNewest - + PreserveNewest - + PreserveNewest - + PreserveNewest - + PreserveNewest - + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + PreserveNewest diff --git a/vcpkg_unix.sh b/vcpkg_unix.sh index e3493776..d0e432ed 100755 --- a/vcpkg_unix.sh +++ b/vcpkg_unix.sh @@ -1,14 +1,25 @@ #!/bin/bash set -e +case ${1:-$(uname -m)} in + x86_64|x64) + arch=x64 + ;; + aarch64|arm64) + arch=arm64 + ;; + *) + echo "Architecture not supported" + exit 1 + ;; +esac + case $(uname) in Linux) - os=Linux - triplet=x64-linux + os=linux ;; Darwin) - os=macOS - triplet=x64-osx + os=osx if ! which brew >/dev/null || [ ! -x $(brew --prefix)/opt/bison/bin/bison ] then echo 'The version of bison provided with macOS is too old.' @@ -25,6 +36,8 @@ case $(uname) in ;; esac +triplet=$arch-$os + # Make sure reading vcpkg_version.txt works even when it doesn't end with a newline read -r vcpkg_url vcpkg_ref << EOF $(cat vcpkg_version.txt) @@ -33,9 +46,19 @@ EOF mkdir -p build cd build # Clone without checking out a branch, as vcpkg_ref could be a commit SHA -git clone $vcpkg_url vcpkg.$os -cd vcpkg.$os +git clone $vcpkg_url vcpkg.$triplet +cd vcpkg.$triplet git checkout $vcpkg_ref + +# Only build release configuration in CI +if [ "$GITHUB_ACTIONS" == "true" ] +then + for triplet_file in triplets/{,community/}$triplet.cmake + do + [ -f "$triplet_file" ] && echo "set(VCPKG_BUILD_TYPE release)" >> "$triplet_file" + done +fi + ./bootstrap-vcpkg.sh ./vcpkg install arrow:$triplet diff --git a/vcpkg_version.txt b/vcpkg_version.txt index c77c82bb..2ee01113 100644 --- a/vcpkg_version.txt +++ b/vcpkg_version.txt @@ -1 +1 @@ -https://github.com/microsoft/vcpkg.git ba69c09fab296d975ee4f578d070f9c67da21dd5 +https://github.com/microsoft/vcpkg.git 77c4e0cb087b3cecb5525dacd1fad18a0bb23573 diff --git a/vcpkg_windows.bat b/vcpkg_windows.bat index e0343cbd..de7c5927 100644 --- a/vcpkg_windows.bat +++ b/vcpkg_windows.bat @@ -3,16 +3,18 @@ for /f "tokens=1,2" %%a in (vcpkg_version.txt) do ( set vcpkg_ref=%%b ) || goto :error +set triplet=x64-windows-static + mkdir build cd build || goto :error rem Clone without checking out a branch, as vcpkg_ref could be a commit SHA -git clone %vcpkg_url% vcpkg.Windows || goto :error -cd vcpkg.Windows || goto :error +git clone %vcpkg_url% vcpkg.%triplet% || goto :error +cd vcpkg.%triplet% || goto :error git checkout %vcpkg_ref% || goto :error -if "%GITHUB_ACTIONS%"=="true" echo set(VCPKG_BUILD_TYPE release) >> triplets\x64-windows-static.cmake || goto :error +if "%GITHUB_ACTIONS%"=="true" echo set(VCPKG_BUILD_TYPE release) >> triplets\%triplet%.cmake || goto :error call .\bootstrap-vcpkg.bat || goto :error -.\vcpkg.exe install arrow:x64-windows-static || goto :error +.\vcpkg.exe install arrow:%triplet% || goto :error cd .. cd ..