Build wheels for Linux ARM64 (#5511)

### What Build linux-aarch64 wheels (with manylinux_2_31) and fixes the C and CLI builds to bring the min required glibc version to 2.31. **Note**: cursory inspection indicates that the _actual_ min glibc version 2.29. However, we don't formally test that. The only test is done by `maturin` on our wheels, against the `manylinux_2_31` standard. Amongst other things, this PR: - updates our ci docker image to be compatible with aarch64 (#5543 – thanks @jleibs) - remove all instances of double GCS authentication (our setup-rust action auth to GCS, so an explicit auth isn't necessary, _and_ would break cleanup in linux-arm/cli build workflow—why there only is unclear) - use it for all linux-aarch64 builds - fixes `pixi.toml` for linux-aarch64 compatibility - bumps pixi to 0.16.1 everywhere in the ci - adds a manual trigger for building wheels - (mostly) unifies the target naming scheme #### Related - Follow-up to: - #5489 - #5503 - Closes #4136 - Fixes #5507 #### Further work - ~~support lower glibc version: #5512 - test more wheels: #5525 ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested the web demo (if applicable): * Using newly built examples: [app.rerun.io](https://app.rerun.io/pr/5511/index.html) * Using examples from latest `main` build: [app.rerun.io](https://app.rerun.io/pr/5511/index.html?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) * Using full set of examples from `nightly` build: [app.rerun.io](https://app.rerun.io/pr/5511/index.html?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG * [x] If applicable, add a new check to the [release checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)! - [PR Build Summary](https://build.rerun.io/pr/5511) - [Docs preview](https://rerun.io/preview/c029b411730d035b9babdba3f9721dbe2905196b/docs)  - [Examples preview](https://rerun.io/preview/c029b411730d035b9babdba3f9721dbe2905196b/examples)  - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html) --------- Co-authored-by: Jeremy Leibs <[email protected]>
rerun-io · Mar 18, 2024 · be1fbcc · be1fbcc
1 parent c8ab078
commit be1fbcc
Show file tree

Hide file tree

Showing 29 changed files with 6,974 additions and 1,067 deletions.
diff --git a/.github/workflows/auto_release_crates.yml b/.github/workflows/auto_release_crates.yml
@@ -22,7 +22,7 @@ jobs:
 
       - uses: prefix-dev/[email protected]
         with:
-          pixi-version: v0.13.0
+          pixi-version: v0.16.1
 
       - name: Install dependencies
         shell: bash

diff --git a/.github/workflows/contrib_checks.yml b/.github/workflows/contrib_checks.yml
@@ -86,7 +86,7 @@ jobs:
     name: Check if running codegen would produce any changes
     runs-on: ubuntu-latest-16-cores
     container:
-      image: rerunio/ci_docker:0.11.0
+      image: rerunio/ci_docker:0.12.0
     steps:
       # Note: We explicitly don't override `ref` here. We need to see if changes would be made
       # in a context where we have merged with main. Otherwise we might miss changes such as one
@@ -101,7 +101,7 @@ jobs:
 
       - uses: prefix-dev/[email protected]
         with:
-          pixi-version: v0.13.0
+          pixi-version: v0.16.1
 
       - name: Codegen check
         shell: bash
@@ -112,7 +112,7 @@ jobs:
     name: Rust lints (fmt, check, cranky, tests, doc)
     runs-on: ubuntu-latest-16-cores
     container:
-      image: rerunio/ci_docker:0.11.0
+      image: rerunio/ci_docker:0.12.0
     steps:
       - uses: actions/checkout@v4
 
@@ -183,7 +183,7 @@ jobs:
     name: Check Rust web build (wasm32 + wasm-bindgen)
     runs-on: ubuntu-latest-16-cores
     container:
-      image: rerunio/ci_docker:0.11.0
+      image: rerunio/ci_docker:0.12.0
     steps:
       - uses: actions/checkout@v4
 
@@ -212,7 +212,7 @@ jobs:
 
       - uses: prefix-dev/[email protected]
         with:
-          pixi-version: v0.13.0
+          pixi-version: v0.16.1
 
       - run: pixi run lint-taplo
         shell: bash
@@ -268,7 +268,7 @@ jobs:
     name: Cargo Deny
     runs-on: ubuntu-latest
     container:
-      image: rerunio/ci_docker:0.11.0
+      image: rerunio/ci_docker:0.12.0
     steps:
       - uses: actions/checkout@v4
 
@@ -294,13 +294,13 @@ jobs:
     name: C++ tests
     runs-on: ubuntu-latest
     container:
-      image: rerunio/ci_docker:0.11.0
+      image: rerunio/ci_docker:0.12.0
     steps:
       - uses: actions/checkout@v4
 
       - uses: prefix-dev/[email protected]
         with:
-          pixi-version: v0.13.0
+          pixi-version: v0.16.1
 
       # TODO(emilk): make this work somehow. Right now this just results in
       # > Compiler: GNU 12.3.0 (/__w/rerun/rerun/.pixi/env/bin/x86_64-conda-linux-gnu-c++)

diff --git a/.github/workflows/contrib_rerun_py.yml b/.github/workflows/contrib_rerun_py.yml
@@ -36,13 +36,13 @@ jobs:
     name: Build Wheels
     runs-on: ubuntu-latest-16-cores
     container:
-      image: rerunio/ci_docker:0.11.0
+      image: rerunio/ci_docker:0.12.0
     steps:
       - uses: actions/checkout@v4
 
       - uses: prefix-dev/[email protected]
         with:
-          pixi-version: v0.13.0
+          pixi-version: v0.16.1
 
       # These should already be in the docker container, but run for good measure. A no-op install
       # should be fast, and this way things don't break if we add new packages without rebuilding

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -53,8 +53,8 @@ jobs:
     uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
     with:
       CONCURRENCY: nightly-linux
-      PLATFORM: linux
-      WHEEL_ARTIFACT_NAME: linux-wheel
+      PLATFORM: linux-x64
+      WHEEL_ARTIFACT_NAME: linux-x64-wheel
       MODE: "pr"
     secrets: inherit
 
@@ -65,7 +65,7 @@ jobs:
     with:
       CONCURRENCY: nightly
       CHANNEL: nightly
-      WHEEL_ARTIFACT_NAME: linux-wheel
+      WHEEL_ARTIFACT_NAME: linux-x64-wheel
     secrets: inherit
 
   upload-examples:

diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml
@@ -76,15 +76,17 @@ jobs:
       FULL: "true"
     secrets: inherit
 
+  # Build and test a single wheel to limit CI cost. We use linux-x64 because it's fast. linux-arm64 would also be a good
+  # choice, but reusable_test_wheels.yml is broken for that target (https://github.com/rerun-io/rerun/issues/5525)
   min-wheel-build:
     name: "Minimum Wheel Build"
     if: github.event.pull_request.head.repo.owner.login == 'rerun-io'
     uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
     with:
       CONCURRENCY: pr-${{ github.event.pull_request.number }}
       MODE: "pr"
-      PLATFORM: linux
-      WHEEL_ARTIFACT_NAME: "linux-wheel-fast"
+      PLATFORM: linux-x64
+      WHEEL_ARTIFACT_NAME: "linux-x64-wheel-fast"
     secrets: inherit
 
   min-wheel-test:
@@ -94,8 +96,8 @@ jobs:
     uses: ./.github/workflows/reusable_test_wheels.yml
     with:
       CONCURRENCY: pr-${{ github.event.pull_request.number }}
-      PLATFORM: linux
-      WHEEL_ARTIFACT_NAME: "linux-wheel-fast"
+      PLATFORM: linux-x64
+      WHEEL_ARTIFACT_NAME: "linux-x64-wheel-fast"
     secrets: inherit
 
   build-js:
@@ -133,7 +135,7 @@ jobs:
     with:
       CONCURRENCY: pr-${{ github.event.pull_request.number }}
       CHANNEL: main
-      WHEEL_ARTIFACT_NAME: linux-wheel-fast
+      WHEEL_ARTIFACT_NAME: linux-x64-wheel-fast
     secrets: inherit
 
   track-sizes:
@@ -163,7 +165,7 @@ jobs:
     uses: ./.github/workflows/reusable_run_notebook.yml
     with:
       CONCURRENCY: pr-${{ github.event.pull_request.number }}
-      WHEEL_ARTIFACT_NAME: linux-wheel-fast
+      WHEEL_ARTIFACT_NAME: linux-x64-wheel-fast
     secrets: inherit
 
   save-pr-summary:

diff --git a/.github/workflows/on_push_main.yml b/.github/workflows/on_push_main.yml
@@ -209,78 +209,90 @@ jobs:
 
   # -----------------------------------------------------------------------------------
 
-  build-wheel-linux:
+  build-wheel-linux-arm64:
     needs: [checks]
-    name: "Linux: Build & Upload Wheels"
+    name: "Linux-arm64: Build & Upload Wheels"
     uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
     with:
-      CONCURRENCY: push-linux-${{ github.ref_name }}
-      PLATFORM: linux
-      WHEEL_ARTIFACT_NAME: linux-wheel
+      CONCURRENCY: push-linux-arm64-${{ github.ref_name }}
+      PLATFORM: linux-arm64
+      WHEEL_ARTIFACT_NAME: linux-arm64-wheel
+      MODE: "pypi"
+    secrets: inherit
+
+  build-wheel-linux-x64:
+    needs: [checks]
+    name: "Linux-x64: Build & Upload Wheels"
+    uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
+    with:
+      CONCURRENCY: push-linux-x64-${{ github.ref_name }}
+      PLATFORM: linux-x64
+      WHEEL_ARTIFACT_NAME: linux-x64-wheel
       MODE: "pypi"
     secrets: inherit
 
-  build-wheel-windows:
+  build-wheel-windows-x64:
     needs: [checks]
-    name: "Windows: Build & Upload Wheels"
+    name: "Windows-x64: Build & Upload Wheels"
     uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
     with:
-      CONCURRENCY: push-windows-${{ github.ref_name }}
-      PLATFORM: windows
-      WHEEL_ARTIFACT_NAME: windows-wheel
+      CONCURRENCY: push-windows-x64-${{ github.ref_name }}
+      PLATFORM: windows-x64
+      WHEEL_ARTIFACT_NAME: windows-x64-wheel
       MODE: "pypi"
     secrets: inherit
 
-  build-wheel-macos-arm:
+  build-wheel-macos-arm64:
     needs: [checks]
-    name: "Macos-Arm: Build & Upload Wheels"
+    name: "Macos-arm64: Build & Upload Wheels"
     uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
     with:
-      CONCURRENCY: push-macos-arm-${{ github.ref_name }}
-      PLATFORM: macos-arm
-      WHEEL_ARTIFACT_NAME: macos-arm-wheel
+      CONCURRENCY: push-macos-arm64-${{ github.ref_name }}
+      PLATFORM: macos-arm64
+      WHEEL_ARTIFACT_NAME: macos-arm64-wheel
       MODE: "pypi"
     secrets: inherit
 
-  build-wheel-macos-intel:
+  build-wheel-macos-x64:
     needs: [checks]
-    name: "Macos-Intel: Build & Upload Wheels"
+    name: "Macos-x64: Build & Upload Wheels"
     uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
     with:
-      CONCURRENCY: push-macos-intel-${{ github.ref_name }}
-      PLATFORM: macos-intel
-      WHEEL_ARTIFACT_NAME: "macos-intel-wheel"
+      CONCURRENCY: push-macos-x64-${{ github.ref_name }}
+      PLATFORM: macos-x64
+      WHEEL_ARTIFACT_NAME: "macos-x64-wheel"
       MODE: "pypi"
     secrets: inherit
 
-  test-wheel-linux:
-    needs: [checks, build-wheel-linux]
-    name: "Linux: Test Wheels"
+  test-wheel-linux-x64:
+    needs: [checks, build-wheel-linux-x64]
+    name: "Linux-x64: Test Wheels"
     uses: ./.github/workflows/reusable_test_wheels.yml
     with:
-      CONCURRENCY: push-linux-${{ github.ref_name }}
-      PLATFORM: linux
-      WHEEL_ARTIFACT_NAME: linux-wheel
+      CONCURRENCY: push-linux-x64-${{ github.ref_name }}
+      PLATFORM: linux-x64
+      WHEEL_ARTIFACT_NAME: linux-x64-wheel
     secrets: inherit
 
-  test-wheel-windows:
-    needs: [checks, build-wheel-windows]
-    name: "Windows: Test Wheels"
+  test-wheel-windows-x64:
+    needs: [checks, build-wheel-windows-x64]
+    name: "Windows-x64: Test Wheels"
     uses: ./.github/workflows/reusable_test_wheels.yml
     with:
-      CONCURRENCY: push-windows-${{ github.ref_name }}
-      PLATFORM: windows
-      WHEEL_ARTIFACT_NAME: windows-wheel
+      CONCURRENCY: push-windows-x64-${{ github.ref_name }}
+      PLATFORM: windows-x64
+      WHEEL_ARTIFACT_NAME: windows-x64-wheel
     secrets: inherit
 
   generate-pip-index:
     name: "Generate Pip Index"
     needs:
       [
-        build-wheel-linux,
-        build-wheel-windows,
-        build-wheel-macos-arm,
-        build-wheel-macos-intel,
+        build-wheel-linux-arm64,
+        build-wheel-linux-x64,
+        build-wheel-macos-arm64,
+        build-wheel-macos-x64,
+        build-wheel-windows-x64,
       ]
     uses: ./.github/workflows/reusable_pip_index.yml
     with:
@@ -291,10 +303,10 @@ jobs:
     name: "Bundle and upload rerun_cpp_sdk.zip"
     needs:
       [
-        build-rerun_c-and-upload-linux-x64,
         build-rerun_c-and-upload-linux-arm64,
-        build-rerun_c-and-upload-macos-x64,
+        build-rerun_c-and-upload-linux-x64,
         build-rerun_c-and-upload-macos-arm64,
+        build-rerun_c-and-upload-macos-x64,
         build-rerun_c-and-upload-windows-x64,
       ]
     uses: ./.github/workflows/reusable_bundle_and_upload_rerun_cpp.yml
@@ -307,19 +319,19 @@ jobs:
       cancel-in-progress: true
     needs:
       [
-        upload-web,
-        generate-pip-index,
-        build-rerun_c-and-upload-linux-x64,
-        build-rerun_c-and-upload-linux-arm64,
-        build-rerun_c-and-upload-macos-x64,
-        build-rerun_c-and-upload-macos-arm64,
-        build-rerun_c-and-upload-windows-x64,
-        build-rerun-cli-and-upload-linux-x64,
         build-rerun-cli-and-upload-linux-arm64,
-        build-rerun-cli-and-upload-macos-x64,
+        build-rerun-cli-and-upload-linux-x64,
         build-rerun-cli-and-upload-macos-arm64,
+        build-rerun-cli-and-upload-macos-x64,
         build-rerun-cli-and-upload-windows-x64,
+        build-rerun_c-and-upload-linux-arm64,
+        build-rerun_c-and-upload-linux-x64,
+        build-rerun_c-and-upload-macos-arm64,
+        build-rerun_c-and-upload-macos-x64,
+        build-rerun_c-and-upload-windows-x64,
         bundle-and-upload-rerun_cpp,
+        generate-pip-index,
+        upload-web,
       ]
     runs-on: "ubuntu-latest"
     steps:

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -90,7 +90,7 @@ jobs:
 
       - uses: prefix-dev/[email protected]
         with:
-          pixi-version: v0.13.0
+          pixi-version: v0.16.1
 
       - name: Update crate versions
         id: versioning
@@ -391,7 +391,7 @@ jobs:
 
       - uses: prefix-dev/[email protected]
         with:
-          pixi-version: v0.13.0
+          pixi-version: v0.16.1
 
       - name: Install dependencies
         shell: bash

diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml
@@ -52,7 +52,7 @@ jobs:
     runs-on: ubuntu-latest-16-cores
 
     container:
-      image: rerunio/ci_docker:0.11.0
+      image: rerunio/ci_docker:0.12.0
 
     steps:
       - uses: actions/checkout@v4