Merge remote-tracking branch 'upstream/main' into fix

.github/workflows/cargo_machete.yml

@@ -0,0 +1,17 @@
+name: Cargo Machete
+
+on:
+  push:
+    branches:
+      - "main"
+  pull_request:
+    types: [opened, synchronize]
+
+jobs:
+  cargo-machete:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Machete
+        uses: bnjbvr/cargo-machete@main

.github/workflows/contrib_checks.yml

@@ -61,21 +61,15 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
+      - uses: prefix-dev/[email protected]
         with:
-          python-version: "3.8"
-
-      - name: Install Python dependencies
-        shell: bash
-        run: |
-          pip install --upgrade pip
-          pip install -r rerun_py/requirements-doc.txt
+          pixi-version: v0.25.0
+          environments: py-docs
 
       - name: Build via mkdocs
         shell: bash
         run: |
-          mkdocs build --strict -f rerun_py/mkdocs.yml
+          pixi run -e py-docs mkdocs build --strict -f rerun_py/mkdocs.yml
 
   no-codegen-changes:
     name: Check if running codegen would produce any changes

.github/workflows/contrib_rerun_py.yml

@@ -61,11 +61,11 @@ jobs:
           # this stops `re_web_viewer_server/build.rs` from running
           RERUN_IS_PUBLISHING: true
         run: |
-          cargo build \
+          pixi run cargo build \
             --locked \
             -p rerun-cli \
             --no-default-features \
-            --features native_viewer,web_viewer \
+            --features release \
             --release \
             --target x86_64-unknown-linux-gnu
 

.github/workflows/labels.yml

@@ -39,4 +39,4 @@ jobs:
         with:
           mode: minimum
           count: 1
-          labels: "📊 analytics, 🟦 blueprint, 🪳 bug, 🌊 C++ API, CLI, codegen/idl, 🧑‍💻 dev experience, dependencies, 📖 documentation, 💬 discussion, examples, exclude from changelog, 🪵 Log-API, 📉 performance, 🐍 Python API, ⛃ re_datastore, 🔍 re_query, 📺 re_viewer, 🔺 re_renderer, 🚜 refactor, ⛴ release, 🦀 Rust API, 🔨 testing, ui, 🕸️ web"
+          labels: "📊 analytics, 🟦 blueprint, 🪳 bug, 🌊 C++ API, CLI, codegen/idl, 🧑‍💻 dev experience, dependencies, 📖 documentation, 💬 discussion, examples, exclude from changelog, 🪵 Log & send APIs, 📉 performance, 🐍 Python API, ⛃ re_datastore, 🔍 re_query, 📺 re_viewer, 🔺 re_renderer, 🚜 refactor, ⛴ release, 🦀 Rust API, 🔨 testing, ui, 🕸️ web"

.github/workflows/nightly.yml

@@ -4,8 +4,8 @@ on:
   workflow_dispatch:
   schedule:
     # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule
-    # 12:15 UTC, every day
-    - cron: "15 12 * * *"
+    # 3am UTC, so 4am-5am CET and evening in east time, basically after everyone's day.
+    - cron: "15 3 * * *"
 
 defaults:
   run:
@@ -59,7 +59,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: 1.76.0
+          toolchain: 1.79.0
 
       - run: cargo build -p rerun
 

.github/workflows/on_push_docs.yml

@@ -59,6 +59,7 @@ jobs:
       - uses: prefix-dev/[email protected]
         with:
           pixi-version: v0.25.0
+          environments: py-docs
 
       - name: Install rerun-sdk
         run: |
@@ -72,11 +73,6 @@ jobs:
           #   because it is not actually going to run
           RUSTFLAGS: "-Awarnings --cfg disable_web_viewer_server"
         run: |
-          # Install latest release version of the Rerun SDK
-          # We don't actually want to build it, because then we'd be installing the dev version.
-          pixi run pip install -r rerun_py/requirements-doc.txt
-          pixi run pip install rerun-sdk==${{ needs.get-version.outputs.version }}
-
           pixi run search-index build \
             landing \
             --url "https://edge.meilisearch.com" \

.github/workflows/release.yml

@@ -234,7 +234,7 @@ jobs:
     secrets: inherit
 
   publish-rerun_c:
-    name: "Build and Publish rerun_c"
+    name: "Build and Publish C/C++ SDKs"
     needs: [version]
     uses: ./.github/workflows/reusable_publish_rerun_c.yml
     with:

.github/workflows/reusable_bench.yml

@@ -155,10 +155,13 @@ jobs:
           save-data-file: false
           auto-push: false
 
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.25.0
+
       - name: Render benchmark result
         if: github.ref == 'refs/heads/main'
         run: |
-          pip install --break-system-package google-cloud-storage==2.9.0
-          scripts/ci/render_bench.py crates \
+          pixi run python scripts/ci/render_bench.py crates \
             --after $(date -d"30 days ago" +%Y-%m-%d) \
             --output "gs://rerun-builds/graphs"

.github/workflows/reusable_build_and_upload_rerun_cli.yml

@@ -176,11 +176,11 @@ jobs:
           # this stops `re_web_viewer_server/build.rs` from running
           RERUN_IS_PUBLISHING: true
         run: |
-          cargo build \
+          pixi run cargo build \
             --locked \
             -p rerun-cli \
             --no-default-features \
-            --features native_viewer,web_viewer \
+            --features release \
             --release \
             --target ${{ needs.set-config.outputs.TARGET }}
 

.github/workflows/reusable_checks_python.yml

@@ -53,20 +53,12 @@ jobs:
         with:
           ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }}
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
+      - uses: prefix-dev/[email protected]
         with:
-          python-version: "3.8"
-          cache: "pip"
-          cache-dependency-path: "rerun_py/requirements-doc.txt"
-
-      - name: Install Python dependencies
-        shell: bash
-        run: |
-          pip install --upgrade pip
-          pip install -r rerun_py/requirements-doc.txt
+          pixi-version: v0.25.0
+          environments: py-docs
 
       - name: Build via mkdocs
         shell: bash
         run: |
-          mkdocs build --strict -f rerun_py/mkdocs.yml
+          pixi run -e py-docs mkdocs build --strict -f rerun_py/mkdocs.yml

.github/workflows/reusable_checks_rust.yml

@@ -100,5 +100,10 @@ jobs:
           workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
           service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
 
-      - name: cargo test --all-targets --all-features
-        run: cargo test --all-targets --all-features
+      # Building with `--all-features` requires extra build tools like `nasm`.
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.25.0
+
+      - name: pixi run cargo test --all-targets --all-features
+        run: pixi run cargo test --all-targets --all-features

.github/workflows/reusable_deploy_docs.yml

@@ -71,18 +71,10 @@ jobs:
           fetch-depth: 0 # Don't do a shallow clone
           ref: ${{ inputs.RELEASE_COMMIT || (github.event_name == 'pull_request' && github.event.pull_request.head.ref || '') }}
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
+      - uses: prefix-dev/[email protected]
         with:
-          python-version: "3.11"
-          cache: "pip"
-          cache-dependency-path: "rerun_py/requirements-doc.txt"
-
-      - name: Install Python dependencies
-        shell: bash
-        run: |
-          pip install --upgrade pip
-          pip install -r rerun_py/requirements-doc.txt
+          pixi-version: v0.25.0
+          environments: py-docs
 
       - name: Set up git author
         run: |
@@ -99,7 +91,7 @@ jobs:
         if: ${{ inputs.UPDATE_LATEST }}
         run: |
           git fetch
-          mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python -u ${{inputs.PY_DOCS_VERSION_NAME}} stable
+          pixi run -e py-docs mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python -u ${{inputs.PY_DOCS_VERSION_NAME}} stable
           git checkout gh-pages
           git checkout --orphan gh-pages-orphan
           git commit -m "Update docs for ${GITHUB_SHA}"
@@ -112,7 +104,7 @@ jobs:
         if: ${{ ! inputs.UPDATE_LATEST }}
         run: |
           git fetch
-          mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python ${{inputs.PY_DOCS_VERSION_NAME}}
+          pixi run -e py-docs mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python ${{inputs.PY_DOCS_VERSION_NAME}}
           git checkout gh-pages
           git checkout --orphan gh-pages-orphan
           git commit -m "Update docs for ${GITHUB_SHA}"
@@ -151,11 +143,15 @@ jobs:
           workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
           service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
 
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.25.0
+
       - name: Delete existing /target/doc
         run: rm -rf ./target/doc
 
       - name: cargo doc --document-private-items
-        run: cargo doc --document-private-items --no-deps --all-features --workspace --exclude rerun-cli
+        run: pixi run cargo doc --document-private-items --no-deps --all-features --workspace --exclude rerun-cli
 
       - name: Set up git author
         run: |

.github/workflows/reusable_publish_rerun_c.yml

@@ -1,4 +1,4 @@
-name: Build and publish C++ SDK
+name: Build and Publish C/C++ SDKs
 
 on:
   workflow_call:

.github/workflows/reusable_test_wheels.yml

@@ -224,3 +224,6 @@ jobs:
         # --no-py-build because rerun-sdk is already built and installed
         # --no-asset-download because we already downloaded the assets
         run: pixi run -e wheel-test-min RUST_LOG=debug python docs/snippets/compare_snippet_output.py --target ${{ needs.set-config.outputs.TARGET }} --no-py-build --no-cpp-build --no-asset-download
+
+      - name: Check the python library signatures
+        run: pixi run -e wheel-test-min python scripts/ci/python_check_signatures.py

.github/workflows/reusable_track_size.yml

@@ -189,10 +189,13 @@ jobs:
           save-data-file: false
           auto-push: false
 
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.25.0
+
       - name: Render benchmark result
         if: github.ref == 'refs/heads/main'
         run: |
-          python3 -m pip install --break-system-package google-cloud-storage==2.9.0
-          scripts/ci/render_bench.py sizes \
+          pixi run python scripts/ci/render_bench.py sizes \
             --after $(date -d"180 days ago" +%Y-%m-%d) \
             --output "gs://rerun-builds/graphs"

.gitignore

@@ -31,6 +31,7 @@ _deps
 # Python build artifacts:
 __pycache__
 *.pyc
+*.pyd
 *.so
 **/.pytest_cache
 

ARCHITECTURE.md

@@ -159,14 +159,15 @@ Update instructions:
 
 ### Application-level store
 
-| Crate                | Description                                                              |
-|----------------------|--------------------------------------------------------------------------|
-| re_entity_db         | In-memory storage of Rerun entities                                      |
-| re_query             | Querying data in the re_chunk_store                                      |
-| re_dataframe         | The Rerun public data APIs.                                              |
-| re_types             | The built-in Rerun data types, component types, and archetypes.          |
-| re_types_blueprint   | The core traits and types that power Rerun's Blueprint sub-system.       |
-| re_log_encoding      | Helpers for encoding and transporting Rerun log messages                 |
+| Crate                 | Description                                                              |
+|-----------------------|--------------------------------------------------------------------------|
+| re_entity_db          | In-memory storage of Rerun entities                                      |
+| re_query              | Querying data in the re_chunk_store                                      |
+| re_dataframe          | The Rerun public data APIs.                                              |
+| re_remote_store_types | Rerun remote store gRPC API types                                        |
+| re_types              | The built-in Rerun data types, component types, and archetypes.          |
+| re_types_blueprint    | The core traits and types that power Rerun's Blueprint sub-system.       |
+| re_log_encoding       | Helpers for encoding and transporting Rerun log messages                 |
 
 
 ### Low-level store
@@ -193,12 +194,13 @@ Update instructions:
 
 ### Build support
 
-| Crate                      | Description                                                      |
-|----------------------------|------------------------------------------------------------------|
-| re_build_info              | Information about the build. Use together with re_build_tools    |
-| re_build_tools             | build.rs helpers for generating build info                       |
-| re_types_builder           | Generates code for Rerun's SDKs from flatbuffers definitions.    |
-| re_dev_tools               | Various tools for Rerun development. Each tool has a subcommand. |
+| Crate                         | Description                                                      |
+|-------------------------------|------------------------------------------------------------------|
+| re_build_info                 | Information about the build. Use together with re_build_tools    |
+| re_build_tools                | build.rs helpers for generating build info                       |
+| re_remote_store_types_builder | Generates code for Rerun remote store gRPC API                   |
+| re_types_builder              | Generates code for Rerun's SDKs from flatbuffers definitions.    |
+| re_dev_tools                  | Various tools for Rerun development. Each tool has a subcommand. |
 
 
 ### Utilities

BUILD.md

@@ -23,12 +23,12 @@ cd rerun
 
 Now install the `pixi` package manager: <https://github.com/prefix-dev/pixi?tab=readme-ov-file#installation>
 
-Make sure `cargo --version` prints `1.76.0` once you are done.
+Make sure `cargo --version` prints `1.79.0` once you are done.
 
 If you are using an Apple-silicon Mac (M1, M2), make sure `rustc -vV` outputs `host: aarch64-apple-darwin`. If not, this should fix it:
 
 ```sh
-rustup set default-host aarch64-apple-darwin && rustup install 1.76.0
+rustup set default-host aarch64-apple-darwin && rustup install 1.79.0
 ```
 
 ## Building and running the Viewer