Merge branch 'cmc/sdk_revamp/2_rust_revamp' into cmc/sdk_revamp/22_ba…

…tching_estimated_size_hell

.github/workflows/manual_release.yml

@@ -0,0 +1,94 @@
+name: Release a branch or target with the given version
+
+on:
+  workflow_dispatch:
+    inputs:
+      RELEASE_VERSION:
+        description: 'Release Version Number (Must match Cargo.toml)'
+        type: string
+        required: true
+      OVERRIDE_COMMIT:
+        description: 'Commit to release'
+        type: string
+        required: false
+
+jobs:
+  publish-wheels:
+    name: 'Publish Wheels'
+
+    permissions:
+      contents: write
+      id-token: "write"
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # Don't do a shallow clone since we need it for finding the full commit hash
+
+      - name: Add SHORT_SHA env property with commit short sha
+        run: |
+          if [ -z "${{ inputs.OVERRIDE_COMMIT }}" ]; then
+            USED_SHA=${{ github.sha }}
+          else
+            USED_SHA=${{ inputs.OVERRIDE_COMMIT }}
+          fi
+          echo "SHORT_SHA=$(echo $USED_SHA | cut -c1-7)" >> $GITHUB_ENV
+
+      - name: Expand short hash to FULL_SHA hash
+        run: |
+          FULL_SHA=$(git rev-parse ${{ env.SHORT_SHA }})
+          echo "FULL_SHA=$FULL_SHA" >> $GITHUB_ENV
+
+      - id: "auth"
+        uses: google-github-actions/auth@v1
+        with:
+          workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
+
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v1'
+        with:
+          version: '>= 363.0.0'
+
+      - name: Install packaging
+        run: |
+          python3 -m pip install packaging google-cloud-storage
+
+      - name: Download wheels from Google Cloud Storage
+        env:
+          BUCKET_PATH:
+        run: |
+          mkdir wheels
+          gsutil cp "gs://rerun-builds/commit/${{ env.SHORT_SHA }}/wheels/*.whl" wheels/
+
+      - name: Verify wheels match the expected release
+        run: |
+          python3 scripts/verify_wheels.py --folder wheels --version ${{ inputs.RELEASE_VERSION }}
+
+      - name: Publish to PyPI
+        uses: PyO3/maturin-action@v1
+        env:
+          # These are both set in the GitHub project configuration
+          MATURIN_REPOSITORY: ${{ vars.PYPI_REPOSITORY }}
+          MATURIN_PYPI_TOKEN: ${{ secrets.MATURIN_PYPI_TOKEN }}
+        with:
+          command: upload
+          args: --skip-existing wheels/*
+
+      # Create the actual prerelease
+      # https://github.com/ncipollo/release-action
+      - name: GitHub Release
+        uses: ncipollo/[email protected]
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          name: "Release - ${{ inputs.RELEASE_VERSION }}"
+          commit: ${{ env.FULL_SHA }}
+          tag: ${{ inputs.RELEASE_VERSION }}
+          artifacts: "wheels/*.whl"
+          generateReleaseNotes: true
+          allowUpdates: true
+          draft: true
+

.github/workflows/reusable_build_and_test_wheels.yml

@@ -203,6 +203,7 @@ jobs:
 
       - name: Check the expected version
         if: ${{ inputs.RELEASE_VERSION != 'prerelease' }}
+        shell: bash
         run: |
           if [ "${{steps.expected_version.outputs.EXPECTED_VERSION}}" != "${{inputs.RELEASE_VERSION}}" ]; then
             echo "Error: EXPECTED_VERSION (${{steps.expected_version.outputs.EXPECTED_VERSION}}) does not match RELEASE_VERSION (${{inputs.RELEASE_VERSION}})"

.github/workflows/reusable_build_web.yml

@@ -44,13 +44,6 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: 1.67.0
-          target: wasm32-unknown-unknown
-          override: true
-
       - name: Set up cargo cache
         uses: Swatinem/rust-cache@v2
         with:

.github/workflows/reusable_checks.yml

@@ -206,13 +206,6 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: default
-          toolchain: 1.67.1
-          target: wasm32-unknown-unknown
-          override: true
-
       - name: Set up cargo cache
         uses: Swatinem/rust-cache@v2
         with:
@@ -249,12 +242,6 @@ jobs:
 
       - uses: actions/checkout@v3
 
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: 1.67.1
-          override: true
-
       - name: Set up cargo cache
         uses: Swatinem/rust-cache@v2
         with:

BUILD.md

@@ -15,15 +15,15 @@ This is a guide to how to build Rerun.
 * Install the Rust toolchain: <https://rustup.rs/>
 * `git clone [email protected]:rerun-io/rerun.git && cd rerun`
 * Run `./scripts/setup_dev.sh`.
-* Make sure `cargo --version` prints `1.67.1` once you are done
+* Make sure `cargo --version` prints `1.69.0` once you are done
 
 
 ### Apple-silicon Macs
 
 If you are using an Apple-silicon Mac (M1, M2), make sure `rustc -vV` outputs `host: aarch64-apple-darwin`. If not, this should fix it:
 
 ```sh
-rustup set default-host aarch64-apple-darwin && rustup install 1.67.1
+rustup set default-host aarch64-apple-darwin && rustup install 1.69.0
 ```
 
 ## Building the docs

Cargo.toml

@@ -15,7 +15,7 @@ homepage = "https://rerun.io"
 include = ["../../LICENSE-APACHE", "../../LICENSE-MIT", "**/*.rs", "Cargo.toml"]
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/rerun-io/rerun"
-rust-version = "1.67"
+rust-version = "1.69"
 version = "0.6.0-alpha.0"
 
 [workspace.dependencies]
@@ -84,9 +84,11 @@ polars-ops = "0.27.1"
 puffin = "0.14"
 smallvec = { version = "1.0", features = ["const_generics", "union"] }
 thiserror = "1.0"
-time = { version = "0.3", features = ["wasm-bindgen"] }
+time = { version = "0.3", default-features = false, features = [
+  "wasm-bindgen",
+] }
 tinyvec = { version = "1.6", features = ["alloc", "rustc_1_55"] }
-tokio = "1.24"
+tokio = { version = "1.24", default-features = false }
 wgpu = { version = "0.16" }
 wgpu-core = { version = "0.16" }
 

ci_docker/Dockerfile

@@ -36,7 +36,7 @@ RUN curl -L https://github.com/NixOS/patchelf/releases/download/0.17.2/patchelf-
 ENV RUSTUP_HOME=/usr/local/rustup \
     CARGO_HOME=/usr/local/cargo \
     PATH=/usr/local/cargo/bin:$PATH \
-    RUST_VERSION=1.67.1 \
+    RUST_VERSION=1.69.0 \
     RUSTUP_VERSION=1.25.2
 
 # Install Rust

clippy.toml

@@ -1,6 +1,6 @@
 # There is also a scripts/clippy_wasm/clippy.toml which forbids some methods that are not available in wasm.
 
-msrv = "1.67"
+msrv = "1.69"
 
 allow-unwrap-in-tests = true
 

crates/re_arrow_store/tests/dump.rs

@@ -236,3 +236,62 @@ fn create_insert_table(ent_path: impl Into<EntityPath>) -> DataTable {
 
     table
 }
+
+// See: https://github.com/rerun-io/rerun/pull/2007
+#[test]
+fn data_store_dump_empty_column() {
+    init_logs();
+
+    // Split tables on 1 row
+    let mut config = re_arrow_store::DataStoreConfig {
+        indexed_bucket_num_rows: 1,
+        ..re_arrow_store::DataStoreConfig::DEFAULT
+    };
+    config.store_insert_ids = false;
+
+    let mut store = DataStore::new(InstanceKey::name(), config);
+
+    data_store_dump_empty_column_impl(&mut store);
+}
+
+fn data_store_dump_empty_column_impl(store: &mut DataStore) {
+    let ent_path: EntityPath = "points".into();
+    let frame1: TimeInt = 1.into();
+    let frame2: TimeInt = 2.into();
+    let frame3: TimeInt = 3.into();
+
+    // Start by inserting a table with 2 rows, one with colors, and one with points.
+    {
+        let (instances1, colors1) = (build_some_instances(3), build_some_colors(3));
+        let row1 = test_row!(ent_path @ [
+                build_frame_nr(frame1),
+            ] => 3; [instances1, colors1]);
+
+        let (instances2, points2) = (build_some_instances(3), build_some_point2d(3));
+        let row2 = test_row!(ent_path @ [
+            build_frame_nr(frame2),
+        ] => 3; [instances2, points2]);
+        let mut table = DataTable::from_rows(TableId::random(), [row1, row2]);
+        table.compute_all_size_bytes();
+        store.insert_table(&table).unwrap();
+    }
+
+    // Now insert another table with points only.
+    {
+        let (instances3, points3) = (build_some_instances(3), build_some_colors(3));
+        let row3 = test_row!(ent_path @ [
+                build_frame_nr(frame3),
+            ] => 3; [instances3, points3]);
+        let mut table = DataTable::from_rows(TableId::random(), [row3]);
+        table.compute_all_size_bytes();
+        store.insert_table(&table).unwrap();
+    }
+
+    let data_msgs: Result<Vec<_>, _> = store
+        .to_data_tables(None)
+        .map(|table| table.to_arrow_msg())
+        .collect();
+
+    // Should end up with 2 tables
+    assert_eq!(data_msgs.unwrap().len(), 2);
+}

crates/re_data_store/src/entity_properties.rs

@@ -141,7 +141,7 @@ impl ExtraQueryHistory {
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
 #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
 pub enum Colormap {
-    /// Perceptually even
+    /// sRGB gray gradient = perceptually even
     Grayscale,
 
     Inferno,

crates/re_log_types/Cargo.toml

@@ -73,10 +73,7 @@ num-derive = "0.3"
 num-traits = "0.2"
 smallvec.workspace = true
 thiserror.workspace = true
-time = { workspace = true, default-features = false, features = [
-  "formatting",
-  "macros",
-] }
+time = { workspace = true, features = ["formatting", "macros"] }
 typenum = "1.15"
 uuid = { version = "1.1", features = ["serde", "v4", "js"] }