wip

.github/workflows/reusable_build_and_test_wheels.yml

@@ -258,8 +258,9 @@ jobs:
         # --release so we can inherit from some of the artifacts that maturin has just built before
         # --target x86_64-unknown-linux-gnu because otherwise cargo loses the target cache... even though this is the target anyhow...
         # --no-py-build because rerun-sdk is already built and installed
-        # NOTE: run with --release so we can reuse some of the "Build Wheel"'s job artifacts, hopefully
-        run: RUST_LOG=debug scripts/ci/run_e2e_roundtrip_tests.py --release --target x86_64-unknown-linux-gnu --no-py-build
+        run: |
+          RUST_LOG=debug tests/roundtrips.py --release --target x86_64-unknown-linux-gnu --no-py-build
+          RUST_LOG=debug docs/code-examples/roundtrips.py --release --target x86_64-unknown-linux-gnu --no-py-build
 
       - name: Cache RRD dataset
         if: needs.set-config.outputs.RUN_TESTS == 'true'

.gitignore

@@ -37,4 +37,4 @@ screenshot*.png
 web_demo
 
 .nox/
-out.rrd
+*.rrd

CMakeLists.txt

@@ -74,3 +74,4 @@ FetchContent_MakeAvailable(LoguruGitRepo) # defines target 'loguru::loguru'
 add_subdirectory(rerun_cpp) # The Rerun C++ SDK library
 add_subdirectory(examples/cpp/minimal)
 add_subdirectory(tests/cpp)
+add_subdirectory(docs/code-examples)

crates/re_components/src/tensor.rs

@@ -26,6 +26,13 @@ impl std::hash::Hash for TensorId {
     }
 }
 
+impl From<uuid::Uuid> for TensorId {
+    #[inline]
+    fn from(value: uuid::Uuid) -> Self {
+        Self(value)
+    }
+}
+
 impl TensorId {
     #[inline]
     pub fn random() -> Self {

crates/re_log_types/src/data_table.rs

@@ -1145,8 +1145,8 @@ impl DataTable {
                             c1.datatype() == c2.datatype(),
                             "Found discrepancy in row #{ri}: cells' datatypes don't match!\n{}",
                             similar_asserts::SimpleDiff::from_str(
-                                &format!("{:?}", c1.datatype()),
-                                &format!("{:?}", c2.datatype()),
+                                &format!("{:?}:{:?}", c1.component_name(), c1.datatype()),
+                                &format!("{:?}:{:?}", c2.component_name(), c2.datatype()),
                                 "cell1",
                                 "cell2"
                             )

crates/re_sdk/src/recording_stream.rs

@@ -20,6 +20,25 @@ use crate::sink::{LogSink, MemorySinkStorage};
 
 // ---
 
+/// Private environment variable meant for tests.
+///
+/// When set, all recording streams will write to disk at the path indicated by the env-var rather
+/// than doing what they were asked to do (`connect()`, `buffered()`, even `save()`..!).
+const ENV_FORCE_SAVE: &str = "_RERUN_TEST_FORCE_SAVE";
+
+/// If private environment variable `_RERUN_TEST_FORCE_SAVE` is set, this will return a `FileSink`
+/// pointing at the path that the env-var is set to.
+///
+/// Useful to save the result of any Rerun snippet to an rrd file, whether it supports it or not.
+fn forced_sink() -> RecordingStreamResult<Option<Box<dyn LogSink>>> {
+    Ok(if let Ok(path) = std::env::var(ENV_FORCE_SAVE) {
+        re_log::warn!(?path, "forcing FileSink");
+        Some(Box::new(crate::sink::FileSink::new(path)?) as Box<dyn LogSink>)
+    } else {
+        None
+    })
+}
+
 /// Errors that can occur when creating/manipulating a [`RecordingStream`].
 #[derive(thiserror::Error, Debug)]
 pub enum RecordingStreamError {
@@ -184,7 +203,11 @@ impl RecordingStreamBuilder {
             RecordingStream::new(
                 store_info,
                 batcher_config,
-                Box::new(crate::log_sink::BufferedSink::new()),
+                if let Some(sink) = forced_sink()? {
+                    sink
+                } else {
+                    Box::new(crate::log_sink::BufferedSink::new())
+                },
             )
         } else {
             re_log::debug!("Rerun disabled - call to buffered() ignored");
@@ -216,7 +239,18 @@ impl RecordingStreamBuilder {
 
         let (enabled, store_info, batcher_config) = self.into_args();
         if enabled {
-            RecordingStream::new(store_info, batcher_config, Box::new(sink)).map(|rec| {
+            RecordingStream::new(
+                store_info,
+                batcher_config,
+                // NOTE: We still have to return a `MemorySinkStorage` that will never get written
+                // to.. beware!
+                if let Some(sink) = forced_sink()? {
+                    sink
+                } else {
+                    Box::new(sink)
+                },
+            )
+            .map(|rec| {
                 storage.rec = Some(rec.clone());
                 (rec, storage)
             })
@@ -250,7 +284,11 @@ impl RecordingStreamBuilder {
             RecordingStream::new(
                 store_info,
                 batcher_config,
-                Box::new(crate::log_sink::TcpSink::new(addr, flush_timeout)),
+                if let Some(sink) = forced_sink()? {
+                    sink
+                } else {
+                    Box::new(crate::log_sink::TcpSink::new(addr, flush_timeout))
+                },
             )
         } else {
             re_log::debug!("Rerun disabled - call to connect() ignored");
@@ -278,7 +316,11 @@ impl RecordingStreamBuilder {
             RecordingStream::new(
                 store_info,
                 batcher_config,
-                Box::new(crate::sink::FileSink::new(path)?),
+                if let Some(sink) = forced_sink()? {
+                    sink
+                } else {
+                    Box::new(crate::sink::FileSink::new(path)?)
+                },
             )
         } else {
             re_log::debug!("Rerun disabled - call to save() ignored");
@@ -324,8 +366,12 @@ impl RecordingStreamBuilder {
     ) -> RecordingStreamResult<RecordingStream> {
         let (enabled, store_info, batcher_config) = self.into_args();
         if enabled {
-            let sink = crate::web_viewer::new_sink(open_browser, bind_ip, web_port, ws_port)
-                .map_err(RecordingStreamError::WebSink)?;
+            let sink = if let Some(sink) = forced_sink()? {
+                sink
+            } else {
+                crate::web_viewer::new_sink(open_browser, bind_ip, web_port, ws_port)
+                    .map_err(RecordingStreamError::WebSink)?
+            };
             RecordingStream::new(store_info, batcher_config, sink)
         } else {
             re_log::debug!("Rerun disabled - call to serve() ignored");
@@ -536,6 +582,7 @@ impl RecordingStream {
         batcher_config: DataTableBatcherConfig,
         sink: Box<dyn LogSink>,
     ) -> RecordingStreamResult<Self> {
+        let sink = forced_sink()?.unwrap_or(sink);
         RecordingStreamInner::new(info, batcher_config, sink).map(|inner| Self {
             inner: Arc::new(Some(inner)),
         })
@@ -1097,7 +1144,12 @@ impl RecordingStream {
     /// terms of data durability and ordering.
     /// See [`Self::set_sink`] for more information.
     pub fn connect(&self, addr: std::net::SocketAddr, flush_timeout: Option<std::time::Duration>) {
-        self.set_sink(Box::new(crate::log_sink::TcpSink::new(addr, flush_timeout)));
+        // NOTE: `forced_sink` is only used for tests, it's ok to unwrap.
+        self.set_sink(if let Some(sink) = forced_sink().unwrap() {
+            sink
+        } else {
+            Box::new(crate::log_sink::TcpSink::new(addr, flush_timeout))
+        });
     }
 
     /// Swaps the underlying sink for a [`crate::sink::MemorySink`] sink and returns the associated
@@ -1109,7 +1161,14 @@ impl RecordingStream {
     pub fn memory(&self) -> MemorySinkStorage {
         let sink = crate::sink::MemorySink::default();
         let buffer = sink.buffer();
-        self.set_sink(Box::new(sink));
+
+        // NOTE: `forced_sink` is only used for tests, it's ok to unwrap.
+        self.set_sink(if let Some(sink) = forced_sink().unwrap() {
+            sink
+        } else {
+            Box::new(sink)
+        });
+
         buffer
     }
 
@@ -1123,7 +1182,14 @@ impl RecordingStream {
         path: impl Into<std::path::PathBuf>,
     ) -> Result<(), crate::sink::FileSinkError> {
         let sink = crate::sink::FileSink::new(path)?;
-        self.set_sink(Box::new(sink));
+
+        // NOTE: `forced_sink` is only used for tests, it's ok to unwrap.
+        self.set_sink(if let Some(sink) = forced_sink().unwrap() {
+            sink
+        } else {
+            Box::new(sink)
+        });
+
         Ok(())
     }
 

crates/re_types/definitions/rerun/archetypes/image.fbs

@@ -19,7 +19,7 @@ namespace rerun.archetypes;
 /// \py -------
 /// \py
 /// \py ```python
-/// \py \include:../../../../../docs/code-examples/image_simple_v2.py
+/// \py \include:../../../../../docs/code-examples/image_simple.py
 /// \py ```
 ///
 /// \rs ## Example

crates/re_types/src/archetypes/image_ext.rs

@@ -43,10 +43,10 @@ impl Image {
         })
     }
 
-    pub fn with_id(self, id: crate::datatypes::TensorId) -> Self {
+    pub fn with_id(self, id: impl Into<crate::datatypes::TensorId>) -> Self {
         Self {
             data: TensorData {
-                id,
+                id: id.into(),
                 shape: self.data.0.shape,
                 buffer: self.data.0.buffer,
             }

crates/re_types/src/datatypes/tensor_id_ext.rs

@@ -21,6 +21,7 @@ impl TensorId {
 }
 
 impl From<uuid::Uuid> for TensorId {
+    #[inline]
     fn from(value: uuid::Uuid) -> Self {
         Self {
             uuid: *value.as_bytes(),

crates/re_types/src/lib.rs

@@ -136,6 +136,7 @@ pub use self::arrow_string::ArrowString;
 
 pub mod external {
     pub use arrow2;
+    pub use uuid;
 }
 
 // TODO(jleibs): Should all of this go into `tensor_data_ext`? Don't have a good way to export

docs/code-examples/CMakeLists.txt

@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.16)
+
+# Setup builds for examples
+file(GLOB sources_list true ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)
+
+add_custom_target(doc_examples)
+
+foreach(SOURCE_PATH ${sources_list})
+    get_filename_component(SOURCE_NAME ${SOURCE_PATH} NAME_WLE)
+
+    if(${SOURCE_NAME} STREQUAL "CMakeFiles")
+        CONTINUE()
+    endif()
+
+    set(EXAMPLE_TARGET doc_example_${SOURCE_NAME})
+
+    add_executable(${EXAMPLE_TARGET} ${SOURCE_PATH})
+
+    set_default_warning_settings(rerun_example)
+    target_link_libraries(${EXAMPLE_TARGET} PRIVATE rerun_sdk)
+
+    add_dependencies(doc_examples ${EXAMPLE_TARGET})
+endforeach()
+

docs/code-examples/README.md

@@ -0,0 +1,30 @@
+# API Examples
+
+These examples showcase common usage of each individual Rerun `Archetype`s.
+
+Most of these example are automatically used as docstrings for the `Archetype`
+APIs in their respective SDKs.
+
+## Usage
+
+You can run each example individually using the following:
+
+- **Python**: `python <example_name>.py`, e.g. `python point3d_random.py`.
+- **Rust**: `cargo r -p code_examples --bin <example_name`, e.g.
+  `cargo r -p code_examples --bin point3d_random`.
+- **C++**:
+  - `./docs/code-examples/build_all_doc_examples.sh` to compile all examples
+  - start a Rerun Viewer listening on the default port: `rerun`
+  - `./build/docs/code-examples/doc_example_<example_name>` to run, e.g.
+    `./build/docs/code-examples/doc_example_point3d_random`
+
+## Roundtrips
+
+All API examples support cross-language roundtrip tests.
+
+These tests check that A) all of our SDKs yield the exact same data when used
+the same way and B) act as regression tests, relying on the fact that it is
+extremely unlikely that all supported languages break in the exact same way at
+the exact same time.
+
+See `./docs/code-examples/roundtrips.py --help`.

docs/code-examples/annotation_context_arrows.cpp

@@ -5,11 +5,11 @@
 namespace rr = rerun;
 
 int main() {
-    auto rr_stream = rr::RecordingStream("rerun_example_annotation_context_rects");
-    rr_stream.connect("127.0.0.1:9876").throw_on_failure();
+    auto rec = rr::RecordingStream("rerun_example_annotation_context_rects");
+    rec.connect("127.0.0.1:9876").throw_on_failure();
 
     // Log an annotation context to assign a label and color to each class
-    rr_stream.log(
+    rec.log(
         "/",
         rr::AnnotationContext({
             rr::datatypes::AnnotationInfo(1, "red", rr::datatypes::Color(255, 0, 0)),
@@ -18,7 +18,7 @@ int main() {
     );
 
     // Log a batch of 2 arrows with different `class_ids`
-    rr_stream.log(
+    rec.log(
         "arrows",
         rr::Arrows3D({{1.0f, 0.0f, 0.0f}, {0.0f, 1.0f, 0.0f}}).with_class_ids({1, 2})
     );