datastore: disable compaction (fixes 2x memory issue) (#1535)

* disable compaction until we support batching

* link

* make it configurable

crates/re_arrow_store/src/store.rs

@@ -178,6 +178,9 @@ pub struct DataStoreConfig {
     ///
     /// See [`DataStore::insert_id_key`].
     pub store_insert_ids: bool,
+
+    /// Should soon-to-be inactive buckets be compacted before being archived?
+    pub enable_compaction: bool,
 }
 
 impl Default for DataStoreConfig {
@@ -193,6 +196,11 @@ impl DataStoreConfig {
         index_bucket_size_bytes: 32 * 1024, // 32kiB
         index_bucket_nb_rows: 1024,
         store_insert_ids: cfg!(debug_assertions),
+        // TODO(cmc): Compaction is disabled until we implement batching.
+        // See https://github.com/rerun-io/rerun/pull/1535 for rationale.
+        //
+        // This has no noticeable impact on performance.
+        enable_compaction: false,
     };
 }
 

crates/re_arrow_store/src/store_write.rs

@@ -1107,8 +1107,9 @@ impl ComponentTable {
                 "allocating new component bucket, previous one overflowed"
             );
 
-            // Archive currently active bucket.
-            active_bucket.archive();
+            if config.enable_compaction {
+                active_bucket.archive();
+            }
 
             let row_offset = active_bucket.row_offset + len;
             self.buckets
@@ -1215,6 +1216,7 @@ impl ComponentBucket {
     /// Archives the bucket as a new one is about to take its place.
     ///
     /// This is a good opportunity to run compaction and other maintenance related tasks.
+    #[allow(dead_code)]
     pub fn archive(&mut self) {
         crate::profile_function!();
 

crates/re_arrow_store/src/test_util.rs

@@ -104,6 +104,7 @@ pub fn all_configs() -> impl Iterator<Item = DataStoreConfig> {
             index_bucket_size_bytes: idx.index_bucket_size_bytes,
             index_bucket_nb_rows: idx.index_bucket_nb_rows,
             store_insert_ids: comp.store_insert_ids || idx.store_insert_ids,
+            enable_compaction: comp.enable_compaction || idx.enable_compaction,
         })
     })
 }