Add MacOS C++ CI, add Linux C++20 CI (#4120)

### What * Fixes #4116 New way of doing test matrices: Load different json files to configure an entire job. -> Much nicer config interface (a separate clean file <3) and less & less arcane job setup \o/ Adds tests for Mac AND Linux C++20! Runs full matrix on main and whenever cpp/hpp/cmake changes are registered. ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested [demo.rerun.io](https://demo.rerun.io/pr/4120) (if applicable) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG - [PR Build Summary](https://build.rerun.io/pr/4120) - [Docs preview](https://rerun.io/preview/0f7f7ebe692424cf6a9e4f269f3b0790c74f9acd/docs)  - [Examples preview](https://rerun.io/preview/0f7f7ebe692424cf6a9e4f269f3b0790c74f9acd/examples)  - [Recent benchmark results](https://ref.rerun.io/dev/bench/) - [Wasm size tracking](https://ref.rerun.io/dev/sizes/)
rerun-io · Nov 2, 2023 · 77926fe · 77926fe
1 parent fb20866
commit 77926fe
Show file tree

Hide file tree

Showing 9 changed files with 190 additions and 81 deletions.
diff --git a/.github/workflows/cpp_matrix_full.json b/.github/workflows/cpp_matrix_full.json
@@ -0,0 +1,28 @@
+{
+    "include": [
+        {
+            "name": "Linux x64, C++17",
+            "runs_on": "ubuntu-latest-16-cores",
+            "cache_key": "build-linux",
+            "extra_env_vars": "RERUN_USE_ASAN=1 RERUN_SET_CXX_VERSION=17"
+        },
+        {
+            "name": "Linux x64, C++20",
+            "runs_on": "ubuntu-latest-16-cores",
+            "cache_key": "build-linux",
+            "extra_env_vars": "RERUN_USE_ASAN=1 RERUN_SET_CXX_VERSION=20"
+        },
+        {
+            "name": "Windows x64",
+            "runs_on": "windows-latest-8-cores",
+            "cache_key": "build-windows",
+            "extra_env_vars": ""
+        },
+        {
+            "name": "Mac aarch64",
+            "runs_on": "macos-latest-large",
+            "cache_key": "build-macos-arm",
+            "extra_env_vars": "RERUN_USE_ASAN=1"
+        }
+    ]
+}
diff --git a/.github/workflows/cpp_matrix_partial.json b/.github/workflows/cpp_matrix_partial.json
@@ -0,0 +1,10 @@
+{
+    "include": [
+        {
+            "name": "Linux x64",
+            "runs_on": "ubuntu-latest-16-cores",
+            "cache_key": "build-linux",
+            "extra_env_vars": ""
+        }
+    ]
+}
diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml
@@ -22,6 +22,33 @@ jobs:
       PR_NUMBER: ${{ github.event.pull_request.number }}
     secrets: inherit
 
+  cpp-paths-filter:
+    runs-on: ubuntu-latest
+    outputs:
+      cpp_changes: ${{ steps.filter.outputs.cpp_changes }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }}
+    - uses: dorny/paths-filter@v2
+      id: filter
+      with:
+        filters: |
+          cpp_changes:
+            - '**/*.hpp'
+            - '**/.cpp'
+            - '**/CMakeLists.txt'
+
+  cpp-tests:
+    name: "C++ tests"
+    # Wait for the cpp-paths-filter to be completed before starting.
+    needs: cpp-paths-filter
+    uses: ./.github/workflows/reusable_checks_cpp.yml
+    with:
+      CONCURRENCY: pr-${{ github.event.pull_request.number }}
+      FULL: "${{ needs.cpp-paths-filter.outputs.cpp_changes }}"
+    secrets: inherit
+
   min-test-wheel:
     name: "Minimum Wheel"
     if: github.event.pull_request.head.repo.owner.login == 'rerun-io'

diff --git a/.github/workflows/on_push_main.yml b/.github/workflows/on_push_main.yml
@@ -13,6 +13,14 @@ jobs:
       CONCURRENCY: push-${{ github.ref_name }}
     secrets: inherit
 
+  cpp_checks:
+    name: Checks
+    uses: ./.github/workflows/reusable_checks_cpp.yml
+    with:
+      CONCURRENCY: push-${{ github.ref_name }}
+      FULL: 'true'
+    secrets: inherit
+
   # Check that a CLEAN container with just `cargo` on it can build rerun:
   clean-build:
     name: cargo build on clean container

diff --git a/.github/workflows/reusable_build_and_upload_rerun_cli.yml b/.github/workflows/reusable_build_and_upload_rerun_cli.yml
@@ -59,7 +59,7 @@ permissions:
 jobs:
   set-config:
     name: Set Config (${{ inputs.PLATFORM }})
-    runs-on: ubuntu-latest-16-cores
+    runs-on: ubuntu-latest
     outputs:
       RUNNER: ${{ steps.set-config.outputs.runner }}
       TARGET: ${{ steps.set-config.outputs.target }}

diff --git a/.github/workflows/reusable_checks.yml b/.github/workflows/reusable_checks.yml
@@ -369,77 +369,3 @@ jobs:
           clang-format-version: "16"
           # Only check c/cpp/h/hpp (default checks also .proto and others)
           include-regex: ^.*\.(c|cpp|h|hpp)$
-
-  cpp-tests-linux:
-    name: C++ tests Linux
-    runs-on: ubuntu-latest
-    container:
-      image: rerunio/ci_docker:0.10.0
-    env:
-      RUSTC_WRAPPER: "sccache"
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }}
-
-      - uses: prefix-dev/[email protected]
-        with:
-          pixi-version: v0.6.0
-          cache: true
-
-      - name: Set up Rust
-        uses: ./.github/actions/setup-rust
-        with:
-          cache_key: "build-linux"
-          # Cache will be produced by `reusable_checks/rs-lints`
-          save_cache: false
-          workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
-          service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
-
-      # TODO(emilk): make this work somehow. Right now this just results in
-      # > Compiler: GNU 12.3.0 (/__w/rerun/rerun/.pixi/env/bin/x86_64-conda-linux-gnu-c++)
-      # 😭
-      # - name: Build and run C++ tests with clang++
-      #   shell: bash
-      #   run: |
-      #     pixi run cpp-clean
-      #     RERUN_WERROR=ON RERUN_USE_ASAN=ON CXX=clang++ pixi run cpp-build-all
-      #     RERUN_WERROR=ON RERUN_USE_ASAN=ON CXX=clang++ pixi run cpp-test
-
-      - name: Build and run C++ tests with g++
-        shell: bash
-        run: |
-          pixi run cpp-clean
-          RERUN_WERROR=ON RERUN_USE_ASAN=ON CXX=g++ pixi run cpp-build-all
-          RERUN_WERROR=ON RERUN_USE_ASAN=ON CXX=g++ pixi run cpp-test
-
-  cpp-tests-windows:
-    name: C++ tests Windows
-    runs-on: windows-latest-8-cores
-    env:
-      RUSTC_WRAPPER: "sccache"
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }}
-
-      - uses: prefix-dev/[email protected]
-        with:
-          pixi-version: v0.6.0
-          cache: true
-
-      - name: Set up Rust
-        uses: ./.github/actions/setup-rust
-        with:
-          cache_key: "build-windows"
-          # Cache will be produced by `reusable_checks/rs-lints`
-          save_cache: false
-          workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
-          service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
-
-      - name: Build and run C++ tests with MSVC
-        shell: bash
-        run: |
-          pixi run cpp-clean
-          RERUN_WERROR=ON pixi run cpp-build-all
-          RERUN_WERROR=ON pixi run cpp-test
diff --git a/.github/workflows/reusable_checks_cpp.yml b/.github/workflows/reusable_checks_cpp.yml
@@ -0,0 +1,92 @@
+name: "C++ Tests on all platforms & compilers"
+
+on:
+  workflow_call:
+    inputs:
+      CONCURRENCY:
+        required: true
+        type: string
+      FULL:
+        # Booleans are almost unusable
+        # See https://medium.com/@sohail.ra5/github-actions-passing-boolean-input-variables-to-reusable-workflow-call-42d39bf7342e
+        type: string
+        required: true
+        default: 'true'
+
+  workflow_dispatch:
+    inputs:
+      CONCURRENCY:
+        required: false
+        type: string
+        default: "adhoc"
+      FULL:
+        description: "If false only linux is tested"
+        type: choice
+        options:
+          - 'true'
+          - 'false'
+
+concurrency:
+  group: ${{ inputs.CONCURRENCY }}-checks_cpp
+  cancel-in-progress: true
+
+env:
+  # See: https://github.com/marketplace/actions/sccache-action
+  SCCACHE_GHA_ENABLED: "false"
+
+permissions:
+  contents: "read"
+  id-token: "write"
+
+jobs:
+  matrix_prep:
+    runs-on: ubuntu-latest
+    outputs:
+      MATRIX: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }}
+      - name: Load C++ test matrix
+        id: set-matrix
+        run: |
+          echo "Full matrix: ${{ inputs.FULL }}"
+          if ${{ inputs.FULL }}; then
+            echo "matrix=$(jq -c . < ./.github/workflows/cpp_matrix_full.json)" >> $GITHUB_OUTPUT
+          else
+            echo "matrix=$(jq -c . < ./.github/workflows/cpp_matrix_partial.json)" >> $GITHUB_OUTPUT
+          fi
+
+  cpp-tests:
+    name: C++ build & test - ${{ matrix.name }}
+    needs: matrix_prep
+    strategy:
+      matrix: ${{ fromJson(needs.matrix_prep.outputs.MATRIX) }}
+    runs-on: ${{ matrix.runs_on }}
+    env:
+      RUSTC_WRAPPER: "sccache"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }}
+
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.6.0
+          cache: true
+
+      - name: Set up Rust
+        uses: ./.github/actions/setup-rust
+        with:
+          cache_key: ${{ matrix.cache_key }}
+          # Cache will be produced by `reusable_checks/rs-lints`
+          save_cache: false
+          workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
+
+      - name: Build and run C++ tests
+        shell: bash
+        run: |
+          pixi run cpp-clean
+          ${{ matrix.extra_env_vars }} RERUN_WERROR=ON pixi run cpp-build-all
+          ${{ matrix.extra_env_vars }} RERUN_WERROR=ON pixi run cpp-test
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -2,6 +2,9 @@ cmake_minimum_required(VERSION 3.16...3.27)
 
 project(rerun_cpp_proj LANGUAGES CXX)
 
+# ------------------------------------------------------------------------------
+
+# Environment driven hooks so we can configure CMake configure even if its buried deep inside a pixi task.
 if(DEFINED ENV{RERUN_WERROR})
     message(STATUS "Compiling with warnings as errors since RERUN_WERROR is set.")
     set(CMAKE_COMPILE_WARNING_AS_ERROR ON)
@@ -12,6 +15,13 @@ if(DEFINED ENV{RERUN_USE_ASAN})
     set(RERUN_USE_ASAN ON)
 endif()
 
+if(DEFINED ENV{RERUN_SET_CXX_VERSION})
+    message(STATUS "Overwriting CMAKE_CXX_STANDARD to $ENV{RERUN_SET_CXX_VERSION} since RERUN_SET_CXX_VERSION is set accordingly.")
+    set(CMAKE_CXX_STANDARD $ENV{RERUN_SET_CXX_VERSION})
+endif()
+
+# ------------------------------------------------------------------------------
+
 function(set_default_warning_settings target)
     if(MSVC)
         # TODO(andreas): Try to enable /Wall
@@ -22,6 +32,10 @@ function(set_default_warning_settings target)
         if(CMAKE_COMPILE_WARNING_AS_ERROR)
             target_compile_options(${target} PRIVATE /WX)
         endif()
+
+        if(RERUN_USE_ASAN)
+            message(WARNING "Rerun C++ build does not support address sanatizer ('asan') on Windows. Ignoring RERUN_USE_ASAN.")
+        endif()
     else()
         # Enabled warnings.
         target_compile_options(${target} PRIVATE
@@ -129,14 +143,14 @@ FetchContent_Declare(LoguruGitRepo
     GIT_REPOSITORY "https://github.com/emilk/loguru" # can be a filesystem path
     GIT_TAG "master"
 )
+FetchContent_MakeAvailable(LoguruGitRepo) # defines target 'loguru::loguru'
 
 # Set any loguru compile-time flags before calling MakeAvailable()
 # Stacktraces are not yet supported on Windows.
 if(NOT WIN32)
     set(LOGURU_STACKTRACES 1)
 endif()
 
-FetchContent_MakeAvailable(LoguruGitRepo) # defines target 'loguru::loguru'
 
 # ------------------------------------------------------------------------------
 add_subdirectory(rerun_cpp) # The Rerun C++ SDK library

diff --git a/rerun_cpp/CMakeLists.txt b/rerun_cpp/CMakeLists.txt
@@ -151,11 +151,15 @@ if(RERUN_DOWNLOAD_AND_BUILD_ARROW)
         endif()
     endif()
 
-    # Enable multithreaded compiling of Arrow on MSVC.
+
     if(MSVC)
-        set(DARROW_CXXFLAGS "/MP")
+        # Enable multithreaded compiling of Arrow on MSVC.
+        set(ARROW_CXXFLAGS "/MP")
+        # ASAN doesn't work with arrow (yet?)
+        set(ARROW_ASAN OFF)
     else()
-        set(DARROW_CXXFLAGS "")
+        set(ARROW_CXXFLAGS "")
+        set(ARROW_ASAN ${RERUN_USE_ASAN})
     endif()
 
     # Workaround for https://github.com/apache/arrow/issues/36117
@@ -191,11 +195,11 @@ if(RERUN_DOWNLOAD_AND_BUILD_ARROW)
         -DARROW_BOOST_USE_SHARED=OFF
         -DARROW_BUILD_SHARED=${ARROW_BUILD_SHARED}
         -DARROW_BUILD_STATIC=${ARROW_BUILD_STATIC}
-        -DARROW_CXXFLAGS=${DARROW_CXXFLAGS}
+        -DARROW_CXXFLAGS=${ARROW_CXXFLAGS}
         -DARROW_IPC=ON
         -DARROW_JEMALLOC=OFF # We encountered some build issues with jemalloc, use mimalloc instead.
         -DARROW_MIMALLOC=ON
-        -DARROW_USE_ASAN=${RERUN_USE_ASAN}
+        -DARROW_USE_ASAN=${ARROW_ASAN}
         -DARROW_USE_TSAN=OFF
         -DARROW_USE_UBSAN=OFF
         -DBOOST_SOURCE=BUNDLED