Introduce re_types_core (#3878)

This PR introduces a new crate: `re_types_core`. `re_types_core` only contains the fundamental traits and types that make up Rerun's data model. It is split off from the existing `re_types`. This makes it possible to work with our data model abstractions without having to depend on the `re_types` behemoth. This is more than a DX improvement: since so many things depend directly or indirectly on `re_types`, it is very easy to end-up with unsolvable dependency cycles. This helps with that in some cases (though certainly not all). In particular, `re_tuid` (and by extension `re_format`) are now completely free of `re_types`. For convenience, `re_types` reexports all of `re_types_core`, so the public API looks unchanged. In a handful of instances (`re_arrow_store`, `re_data_store`, `re_log_types`, `re_query`), I've went the extra mile and started porting these crates towards raw `re_types_core` rather than relying on the reexports. The reason is that, upon closer inspection, these crates are very close to being able to live free of `re_types`. In the future, the custom crate and custom module attributes coming with #3741 might allow us to make these independent. Similarly, the codegen now uses `re_types_core` directly, as that makes the life of the upcoming "serde-codegen" work much easier.
rerun-io · Oct 16, 2023 · 11dee21 · 11dee21
1 parent f292a5c
commit 11dee21
Show file tree

Hide file tree

Showing 210 changed files with 3,396 additions and 2,911 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -58,6 +58,7 @@ re_tensor_ops = { path = "crates/re_tensor_ops", version = "=0.10.0-alpha.7", de
 re_time_panel = { path = "crates/re_time_panel", version = "=0.10.0-alpha.7", default-features = false }
 re_tracing = { path = "crates/re_tracing", version = "=0.10.0-alpha.7", default-features = false }
 re_tuid = { path = "crates/re_tuid", version = "=0.10.0-alpha.7", default-features = false }
+re_types_core = { path = "crates/re_types_core", version = "=0.10.0-alpha.7", default-features = false }
 re_types = { path = "crates/re_types", version = "=0.10.0-alpha.7", default-features = false }
 re_types_builder = { path = "crates/re_types_builder", version = "=0.10.0-alpha.7", default-features = false }
 re_ui = { path = "crates/re_ui", version = "=0.10.0-alpha.7", default-features = false }
@@ -158,9 +159,12 @@ puffin_http = "0.13"
 rand = { version = "0.8", default-features = false }
 rayon = "1.7"
 rfd = { version = "0.12", default_features = false, features = ["xdg-portal"] }
+rmp-serde = "1"
 ron = "0.8.0"
 serde = "1"
+serde_bytes = "0.11"
 serde_json = { version = "1", default-features = false, features = ["std"] }
+serde_test = "1"
 serde_yaml = { version = "0.9.21", default-features = false }
 sha2 = "0.10"
 similar-asserts = "1.4.2"

diff --git a/crates/re_arrow_store/Cargo.toml b/crates/re_arrow_store/Cargo.toml
@@ -40,6 +40,8 @@ re_format.workspace = true
 re_log_types.workspace = true
 re_log.workspace = true
 re_tracing.workspace = true
+# NOTE: We're on our way to make this crate re_types-free, which is why we import both.
+re_types_core.workspace = true
 re_types.workspace = true
 
 # External dependencies:

diff --git a/crates/re_arrow_store/benches/arrow2.rs b/crates/re_arrow_store/benches/arrow2.rs
@@ -9,13 +9,13 @@ use arrow2::array::{Array, PrimitiveArray, StructArray, UnionArray};
 use criterion::Criterion;
 use itertools::Itertools;
 
-use re_log_types::{DataCell, SizeBytes as _};
+use re_log_types::DataCell;
 use re_types::datagen::{build_some_instances, build_some_positions2d};
 use re_types::{
     components::{InstanceKey, Position2D},
     testing::{build_some_large_structs, LargeStruct},
-    Component,
 };
+use re_types_core::{Component, SizeBytes as _};
 
 // ---
 

diff --git a/crates/re_arrow_store/benches/data_store.rs b/crates/re_arrow_store/benches/data_store.rs
@@ -15,8 +15,8 @@ use re_types::datagen::build_some_instances;
 use re_types::{
     components::InstanceKey,
     testing::{build_some_large_structs, LargeStruct},
-    ComponentName, Loggable as _,
 };
+use re_types_core::{ComponentName, Loggable as _};
 
 criterion_group!(benches, insert, latest_at, latest_at_missing, range, gc);
 criterion_main!(benches);

diff --git a/crates/re_arrow_store/examples/dump_dataframe.rs b/crates/re_arrow_store/examples/dump_dataframe.rs
@@ -7,7 +7,8 @@
 use re_arrow_store::{test_row, DataStore};
 use re_log_types::{build_frame_nr, build_log_time, EntityPath, Time};
 use re_types::datagen::{build_some_instances, build_some_instances_from, build_some_positions2d};
-use re_types::{components::InstanceKey, testing::build_some_large_structs, Loggable};
+use re_types::{components::InstanceKey, testing::build_some_large_structs};
+use re_types_core::Loggable as _;
 
 // ---
 

diff --git a/crates/re_arrow_store/examples/latest_component.rs b/crates/re_arrow_store/examples/latest_component.rs
@@ -11,8 +11,8 @@ use re_types::datagen::build_some_positions2d;
 use re_types::{
     components::{InstanceKey, Position2D},
     testing::{build_some_large_structs, LargeStruct},
-    Loggable,
 };
+use re_types_core::Loggable as _;
 
 fn main() {
     let mut store = DataStore::new(InstanceKey::name(), Default::default());

diff --git a/crates/re_arrow_store/examples/latest_components.rs b/crates/re_arrow_store/examples/latest_components.rs
@@ -13,8 +13,8 @@ use re_types::datagen::build_some_positions2d;
 use re_types::{
     components::{InstanceKey, Position2D},
     testing::{build_some_large_structs, LargeStruct},
-    Loggable,
 };
+use re_types_core::Loggable as _;
 
 fn main() {
     let mut store = DataStore::new(InstanceKey::name(), Default::default());

diff --git a/crates/re_arrow_store/examples/range_components.rs b/crates/re_arrow_store/examples/range_components.rs
@@ -11,8 +11,8 @@ use re_types::datagen::build_some_positions2d;
 use re_types::{
     components::{InstanceKey, Position2D},
     testing::{build_some_large_structs, LargeStruct},
-    Loggable as _,
 };
+use re_types_core::Loggable as _;
 
 fn main() {
     let mut store = DataStore::new(InstanceKey::name(), Default::default());

diff --git a/crates/re_arrow_store/src/polars_util.rs b/crates/re_arrow_store/src/polars_util.rs
@@ -2,7 +2,7 @@ use itertools::Itertools;
 use polars_core::{prelude::*, series::Series};
 use polars_ops::prelude::*;
 use re_log_types::{DataCell, EntityPath, RowId, TimeInt};
-use re_types::ComponentName;
+use re_types_core::ComponentName;
 
 use crate::{ArrayExt, DataStore, LatestAtQuery, RangeQuery};
 

diff --git a/crates/re_arrow_store/src/store.rs b/crates/re_arrow_store/src/store.rs
@@ -5,12 +5,12 @@ use ahash::HashMap;
 use arrow2::datatypes::DataType;
 use nohash_hasher::IntMap;
 use parking_lot::RwLock;
-use re_types::{ComponentName, ComponentNameSet};
+use re_types_core::{ComponentName, ComponentNameSet, SizeBytes};
 use smallvec::SmallVec;
 
 use re_log_types::{
     DataCell, DataCellColumn, EntityPath, EntityPathHash, ErasedTimeVec, NumInstancesVec, RowId,
-    RowIdVec, SizeBytes, TimeInt, TimePoint, TimeRange, Timeline,
+    RowIdVec, TimeInt, TimePoint, TimeRange, Timeline,
 };
 
 // --- Data store ---
@@ -285,7 +285,7 @@ impl DataStore {
         &self.config
     }
 
-    /// Lookup the arrow [`DataType`] of a [`re_types::Component`] in the internal
+    /// Lookup the arrow [`DataType`] of a [`re_types_core::Component`] in the internal
     /// `DataTypeRegistry`.
     pub fn lookup_datatype(&self, component: &ComponentName) -> Option<&DataType> {
         self.type_registry.get(component)
@@ -336,10 +336,10 @@ impl DataStore {
 #[test]
 fn datastore_internal_repr() {
     use re_log_types::DataTable;
-    use re_types::{components::InstanceKey, Loggable as _};
+    use re_types_core::Loggable as _;
 
     let mut store = DataStore::new(
-        InstanceKey::name(),
+        re_types::components::InstanceKey::name(),
         DataStoreConfig {
             indexed_bucket_num_rows: 0,
             store_insert_ids: true,

diff --git a/crates/re_arrow_store/src/store_arrow.rs b/crates/re_arrow_store/src/store_arrow.rs
@@ -6,7 +6,7 @@ use re_log_types::{
     DataCellColumn, DataTable, DataTableResult, RowId, Timeline, COLUMN_INSERT_ID,
     COLUMN_NUM_INSTANCES,
 };
-use re_types::ComponentName;
+use re_types_core::ComponentName;
 
 use crate::store::{IndexedBucket, IndexedBucketInner, PersistentIndexedTable};
 

diff --git a/crates/re_arrow_store/src/store_format.rs b/crates/re_arrow_store/src/store_format.rs
@@ -1,5 +1,5 @@
 use re_format::{format_bytes, format_number};
-use re_log_types::SizeBytes as _;
+use re_types_core::SizeBytes as _;
 
 use crate::{DataStore, IndexedBucket, IndexedTable, PersistentIndexedTable};
 

diff --git a/crates/re_arrow_store/src/store_gc.rs b/crates/re_arrow_store/src/store_gc.rs
@@ -1,8 +1,8 @@
 use ahash::{HashMap, HashSet};
 
 use nohash_hasher::IntMap;
-use re_log_types::{EntityPathHash, RowId, SizeBytes as _, TimeInt, TimeRange, Timeline};
-use re_types::ComponentName;
+use re_log_types::{EntityPathHash, RowId, TimeInt, TimeRange, Timeline};
+use re_types_core::{ComponentName, SizeBytes as _};
 
 use crate::{
     store::{IndexedBucketInner, IndexedTable, PersistentIndexedTable},

diff --git a/crates/re_arrow_store/src/store_helpers.rs b/crates/re_arrow_store/src/store_helpers.rs
@@ -1,6 +1,6 @@
 use re_log_types::{DataCell, DataRow, EntityPath, RowId, TimePoint, Timeline};
 
-use re_types::{Component, ComponentName};
+use re_types_core::{Component, ComponentName};
 
 use crate::{DataStore, LatestAtQuery};
 
@@ -35,7 +35,7 @@ impl<C: Component> std::ops::Deref for VersionedComponent<C> {
 }
 
 impl DataStore {
-    /// Get the latest value for a given [`re_types::Component`] and the associated [`RowId`].
+    /// Get the latest value for a given [`re_types_core::Component`] and the associated [`RowId`].
     ///
     /// This assumes that the row we get from the store only contains a single instance for this
     /// component; it will log a warning otherwise.
@@ -110,7 +110,7 @@ impl DataStore {
         None
     }
 
-    /// Get the latest value for a given [`re_types::Component`] and the associated [`RowId`], assuming it is timeless.
+    /// Get the latest value for a given [`re_types_core::Component`] and the associated [`RowId`], assuming it is timeless.
     ///
     /// This assumes that the row we get from the store only contains a single instance for this
     /// component; it will log a warning otherwise.
@@ -132,7 +132,7 @@ impl DataStore {
 // --- Write ---
 
 impl DataStore {
-    /// Stores a single value for a given [`re_types::Component`].
+    /// Stores a single value for a given [`re_types_core::Component`].
     ///
     /// This is a best-effort helper, it will merely log errors on failure.
     pub fn insert_component<'a, C>(
@@ -172,7 +172,7 @@ impl DataStore {
         }
     }
 
-    /// Stores a single empty value for a given [`re_log_types::ComponentName`].
+    /// Stores a single empty value for a given [`re_types_core::ComponentName`].
     ///
     /// This is a best-effort helper, it will merely log errors on failure.
     pub fn insert_empty_component(

diff --git a/crates/re_arrow_store/src/store_polars.rs b/crates/re_arrow_store/src/store_polars.rs
@@ -10,7 +10,7 @@ use arrow2::{
 };
 use polars_core::{functions::diag_concat_df, prelude::*};
 use re_log_types::{DataCell, DataTable};
-use re_types::ComponentName;
+use re_types_core::ComponentName;
 
 use crate::{
     store::InsertIdVec, ArrayExt, DataStore, DataStoreConfig, IndexedBucket, IndexedBucketInner,

diff --git a/crates/re_arrow_store/src/store_read.rs b/crates/re_arrow_store/src/store_read.rs
@@ -3,7 +3,7 @@ use std::{ops::RangeBounds, sync::atomic::Ordering};
 use itertools::Itertools;
 use re_log::trace;
 use re_log_types::{DataCell, EntityPath, RowId, TimeInt, TimePoint, TimeRange, Timeline};
-use re_types::{ComponentName, ComponentNameSet};
+use re_types_core::{ComponentName, ComponentNameSet};
 use smallvec::SmallVec;
 
 use crate::{DataStore, IndexedBucket, IndexedBucketInner, IndexedTable, PersistentIndexedTable};
@@ -190,7 +190,7 @@ impl DataStore {
     /// ```rust
     /// # use polars_core::{prelude::*, series::Series};
     /// # use re_log_types::{EntityPath, RowId, TimeInt};
-    /// # use re_types::{ComponentName};
+    /// # use re_types_core::{ComponentName};
     /// # use re_arrow_store::{DataStore, LatestAtQuery, RangeQuery};
     /// #
     /// pub fn latest_component(
@@ -359,7 +359,7 @@ impl DataStore {
     /// # use polars_core::{prelude::*, series::Series};
     /// # use re_log_types::{DataCell, EntityPath, RowId, TimeInt};
     /// # use re_arrow_store::{DataStore, LatestAtQuery, RangeQuery};
-    /// # use re_types::ComponentName;
+    /// # use re_types_core::ComponentName;
     /// #
     /// # pub fn dataframe_from_cells<const N: usize>(
     /// #     cells: [Option<DataCell>; N],

diff --git a/crates/re_arrow_store/src/store_sanity.rs b/crates/re_arrow_store/src/store_sanity.rs
@@ -1,7 +1,5 @@
-use re_log_types::{
-    DataCellColumn, RowId, SizeBytes as _, TimeRange, COLUMN_NUM_INSTANCES, COLUMN_TIMEPOINT,
-};
-use re_types::{ComponentName, Loggable};
+use re_log_types::{DataCellColumn, RowId, TimeRange, COLUMN_NUM_INSTANCES, COLUMN_TIMEPOINT};
+use re_types_core::{ComponentName, Loggable, SizeBytes as _};
 
 use crate::{DataStore, IndexedBucket, IndexedBucketInner, IndexedTable, PersistentIndexedTable};
 

diff --git a/crates/re_arrow_store/src/store_stats.rs b/crates/re_arrow_store/src/store_stats.rs
@@ -1,6 +1,6 @@
 use nohash_hasher::IntMap;
-use re_log_types::{SizeBytes, TimePoint, TimeRange};
-use re_types::ComponentName;
+use re_log_types::{TimePoint, TimeRange};
+use re_types_core::{ComponentName, SizeBytes};
 
 use crate::{
     store::IndexedBucketInner, ClusterCellCache, DataStore, DataTypeRegistry, IndexedBucket,

diff --git a/crates/re_arrow_store/src/store_write.rs b/crates/re_arrow_store/src/store_write.rs
@@ -6,10 +6,11 @@ use smallvec::SmallVec;
 
 use re_log::{debug, trace};
 use re_log_types::{
-    DataCell, DataCellColumn, DataCellError, DataRow, DataTable, RowId, SizeBytes as _, TimeInt,
-    TimePoint, TimeRange,
+    DataCell, DataCellColumn, DataCellError, DataRow, DataTable, RowId, TimeInt, TimePoint,
+    TimeRange,
 };
-use re_types::{components::InstanceKey, ComponentName, ComponentNameSet, Loggable};
+use re_types::components::InstanceKey;
+use re_types_core::{ComponentName, ComponentNameSet, Loggable, SizeBytes as _};
 
 use crate::{
     store::MetadataRegistry, DataStore, DataStoreConfig, IndexedBucket, IndexedBucketInner,

diff --git a/crates/re_arrow_store/tests/correctness.rs b/crates/re_arrow_store/tests/correctness.rs
@@ -13,8 +13,9 @@ use re_arrow_store::{
 use re_log_types::{
     build_frame_nr, build_log_time, DataCell, Duration, EntityPath, Time, TimeType, Timeline,
 };
+use re_types::components::InstanceKey;
 use re_types::datagen::{build_some_colors, build_some_instances, build_some_positions2d};
-use re_types::{components::InstanceKey, Loggable as _};
+use re_types_core::Loggable as _;
 
 // ---