From 778ee548886014f79d0edd7095b61ccda9456757 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Mon, 4 Mar 2024 15:52:36 +0100 Subject: [PATCH 1/5] expose/integrate dataloader settings to cpp --- crates/rerun_c/src/lib.rs | 28 ++++++++--- crates/rerun_c/src/rerun.h | 30 +++++++++++- rerun_cpp/src/rerun/c/rerun.h | 30 +++++++++++- rerun_cpp/src/rerun/recording_stream.cpp | 11 ++++- rerun_cpp/src/rerun/recording_stream.hpp | 59 +++++++++++++++++++++--- 5 files changed, 140 insertions(+), 18 deletions(-) diff --git a/crates/rerun_c/src/lib.rs b/crates/rerun_c/src/lib.rs index 9e018e93702e..e814a9dd530c 100644 --- a/crates/rerun_c/src/lib.rs +++ b/crates/rerun_c/src/lib.rs @@ -731,13 +731,16 @@ pub unsafe extern "C" fn rr_recording_stream_log( fn rr_log_file_from_path_impl( stream: CRecordingStream, filepath: CStringView, + entity_path_prefix: CStringView, + timeless: bool, ) -> Result<(), CError> { let stream = recording_stream(stream)?; let filepath = filepath.as_str("filepath")?; + let entity_path_prefix = entity_path_prefix.as_str("entity_path_prefix").ok(); + stream - // TODO(cmc): expose settings - .log_file_from_path(filepath, None, true) + .log_file_from_path(filepath, entity_path_prefix.map(Into::into), timeless) .map_err(|err| { CError::new( CErrorCode::RecordingStreamRuntimeFailure, @@ -753,9 +756,11 @@ fn rr_log_file_from_path_impl( pub unsafe extern "C" fn rr_recording_stream_log_file_from_path( stream: CRecordingStream, filepath: CStringView, + entity_path_prefix: CStringView, + timeless: bool, error: *mut CError, ) { - if let Err(err) = rr_log_file_from_path_impl(stream, filepath) { + if let Err(err) = rr_log_file_from_path_impl(stream, filepath, entity_path_prefix, timeless) { err.write_error(error); } } @@ -766,15 +771,22 @@ fn rr_log_file_from_contents_impl( stream: CRecordingStream, filepath: CStringView, contents: CBytesView, + entity_path_prefix: CStringView, + timeless: bool, ) -> Result<(), CError> { let stream = recording_stream(stream)?; let filepath = filepath.as_str("filepath")?; let contents = contents.as_bytes("contents")?; + let entity_path_prefix = entity_path_prefix.as_str("entity_path_prefix").ok(); stream - // TODO(cmc): expose settings - .log_file_from_contents(filepath, std::borrow::Cow::Borrowed(contents), None, true) + .log_file_from_contents( + filepath, + std::borrow::Cow::Borrowed(contents), + entity_path_prefix.map(Into::into), + timeless, + ) .map_err(|err| { CError::new( CErrorCode::RecordingStreamRuntimeFailure, @@ -791,9 +803,13 @@ pub unsafe extern "C" fn rr_recording_stream_log_file_from_contents( stream: CRecordingStream, filepath: CStringView, contents: CBytesView, + entity_path_prefix: CStringView, + timeless: bool, error: *mut CError, ) { - if let Err(err) = rr_log_file_from_contents_impl(stream, filepath, contents) { + if let Err(err) = + rr_log_file_from_contents_impl(stream, filepath, contents, entity_path_prefix, timeless) + { err.write_error(error); } } diff --git a/crates/rerun_c/src/rerun.h b/crates/rerun_c/src/rerun.h index 7f1ab264501a..fd2b0afd0811 100644 --- a/crates/rerun_c/src/rerun.h +++ b/crates/rerun_c/src/rerun.h @@ -141,6 +141,30 @@ typedef struct rr_spawn_options { rr_string executable_path; } rr_spawn_options; +/// Recommended settings for the [`DataLoader`]. +/// +/// The loader is free to ignore some or all of these. +/// +/// Refer to the field-level documentation for more information about each individual options. +// +// TODO(#3841): expose timepoint settings once we implement stateless APIs +typedef struct rr_data_loader_settings { + /// The recommended `RecordingId` to log the data to. + /// + /// Unspecified by default. + rr_string recording_id; + + /// What should the logged entity paths be prefixed with? + /// + /// Unspecified by default. + rr_string entity_path_prefix; + + /// Should the logged data be timeless? + /// + /// Defaults to `false` if not set. + bool timeless; +} rr_data_loader_settings; + typedef struct rr_store_info { /// The user-chosen name of the application doing the logging. rr_string application_id; @@ -432,7 +456,8 @@ extern void rr_recording_stream_log( /// /// See for more information. extern void rr_recording_stream_log_file_from_path( - rr_recording_stream stream, rr_string path, rr_error* error + rr_recording_stream stream, rr_string path, rr_string entity_path_prefix, bool timeless, + rr_error* error ); /// Logs the given `contents` using all `DataLoader`s available. @@ -444,7 +469,8 @@ extern void rr_recording_stream_log_file_from_path( /// /// See for more information. extern void rr_recording_stream_log_file_from_contents( - rr_recording_stream stream, rr_string path, rr_bytes contents, rr_error* error + rr_recording_stream stream, rr_string path, rr_bytes contents, rr_string entity_path_prefix, + bool timeless, rr_error* error ); // ---------------------------------------------------------------------------- diff --git a/rerun_cpp/src/rerun/c/rerun.h b/rerun_cpp/src/rerun/c/rerun.h index 1158f00ee31f..425c8423f008 100644 --- a/rerun_cpp/src/rerun/c/rerun.h +++ b/rerun_cpp/src/rerun/c/rerun.h @@ -141,6 +141,30 @@ typedef struct rr_spawn_options { rr_string executable_path; } rr_spawn_options; +/// Recommended settings for the [`DataLoader`]. +/// +/// The loader is free to ignore some or all of these. +/// +/// Refer to the field-level documentation for more information about each individual options. +// +// TODO(#3841): expose timepoint settings once we implement stateless APIs +typedef struct rr_data_loader_settings { + /// The recommended `RecordingId` to log the data to. + /// + /// Unspecified by default. + rr_string recording_id; + + /// What should the logged entity paths be prefixed with? + /// + /// Unspecified by default. + rr_string entity_path_prefix; + + /// Should the logged data be timeless? + /// + /// Defaults to `false` if not set. + bool timeless; +} rr_data_loader_settings; + typedef struct rr_store_info { /// The user-chosen name of the application doing the logging. rr_string application_id; @@ -432,7 +456,8 @@ extern void rr_recording_stream_log( /// /// See for more information. extern void rr_recording_stream_log_file_from_path( - rr_recording_stream stream, rr_string path, rr_error* error + rr_recording_stream stream, rr_string path, rr_string entity_path_prefix, bool timeless, + rr_error* error ); /// Logs the given `contents` using all `DataLoader`s available. @@ -444,7 +469,8 @@ extern void rr_recording_stream_log_file_from_path( /// /// See for more information. extern void rr_recording_stream_log_file_from_contents( - rr_recording_stream stream, rr_string path, rr_bytes contents, rr_error* error + rr_recording_stream stream, rr_string path, rr_bytes contents, rr_string entity_path_prefix, + bool timeless, rr_error* error ); // ---------------------------------------------------------------------------- diff --git a/rerun_cpp/src/rerun/recording_stream.cpp b/rerun_cpp/src/rerun/recording_stream.cpp index 0c45d32990ae..9edbea86c4c4 100644 --- a/rerun_cpp/src/rerun/recording_stream.cpp +++ b/rerun_cpp/src/rerun/recording_stream.cpp @@ -257,7 +257,9 @@ namespace rerun { return status; } - Error RecordingStream::try_log_file_from_path(const std::filesystem::path& filepath) const { + Error RecordingStream::try_log_file_from_path( + const std::filesystem::path& filepath, std::string_view entity_path_prefix, bool timeless + ) const { if (!is_enabled()) { return Error::ok(); } @@ -266,6 +268,8 @@ namespace rerun { rr_recording_stream_log_file_from_path( _id, detail::to_rr_string(filepath.string()), + detail::to_rr_string(entity_path_prefix), + timeless, &status ); @@ -273,7 +277,8 @@ namespace rerun { } Error RecordingStream::try_log_file_from_contents( - const std::filesystem::path& filepath, const std::byte* contents, size_t contents_size + const std::filesystem::path& filepath, const std::byte* contents, size_t contents_size, + std::string_view entity_path_prefix, bool timeless ) const { if (!is_enabled()) { return Error::ok(); @@ -288,6 +293,8 @@ namespace rerun { _id, detail::to_rr_string(filepath.string()), data, + detail::to_rr_string(entity_path_prefix), + timeless, &status ); diff --git a/rerun_cpp/src/rerun/recording_stream.hpp b/rerun_cpp/src/rerun/recording_stream.hpp index b8c7a4424c57..90b123c35093 100644 --- a/rerun_cpp/src/rerun/recording_stream.hpp +++ b/rerun_cpp/src/rerun/recording_stream.hpp @@ -3,6 +3,7 @@ #include #include // uint32_t etc. #include +#include #include #include @@ -402,6 +403,27 @@ namespace rerun { return try_log_with_timeless(entity_path, true, archetypes_or_collectiones...); } + /// Logs one or more archetype and/or component batches optionally timeless, returning an error. + /// + /// See `log`/`log_timeless` for more information. + /// Returns an error if an error occurs during serialization or logging. + /// + /// \param entity_path Path to the entity in the space hierarchy. + /// \param timeless If true, the logged components will be timeless. + /// Otherwise, the data will be timestamped automatically with `log_time` and `log_tick`. + /// Additional timelines set by `set_time_sequence` or `set_time` will also be included. + /// \param archetypes_or_collectiones Any type for which the `AsComponents` trait is implemented. + /// This is the case for any archetype or `std::vector`/`std::array`/C-array of components implements. + /// \returns An error if an error occurs during serialization or logging. + /// + /// @see log, try_log, log_timeless, try_log_timeless + template + void log_with_timeless( + std::string_view entity_path, bool timeless, const Ts&... archetypes_or_collectiones + ) const { + try_log_with_timeless(entity_path, timeless, archetypes_or_collectiones...).handle(); + } + /// Logs one or more archetype and/or component batches optionally timeless, returning an error. /// /// See `log`/`log_timeless` for more information. @@ -505,10 +527,16 @@ namespace rerun { /// See for more information. /// /// \param filepath Path to the file to be logged. + /// \param recording_id The recommended `RecordingId` to log the data to. + /// \param entity_path_prefix What should the logged entity paths be prefixed with? + /// \param timeless Should the logged data be timeless? /// /// \see `try_log_file_from_path` - void log_file_from_path(const std::filesystem::path& filepath) const { - try_log_file_from_path(filepath).handle(); + void log_file_from_path( + const std::filesystem::path& filepath, + std::string_view entity_path_prefix = std::string_view(), bool timeless = false + ) const { + try_log_file_from_path(filepath, entity_path_prefix, timeless).handle(); } /// Logs the file at the given `path` using all `DataLoader`s available. @@ -521,9 +549,14 @@ namespace rerun { /// See for more information. /// /// \param filepath Path to the file to be logged. + /// \param entity_path_prefix What should the logged entity paths be prefixed with? + /// \param timeless Should the logged data be timeless? /// /// \see `log_file_from_path` - Error try_log_file_from_path(const std::filesystem::path& filepath) const; + Error try_log_file_from_path( + const std::filesystem::path& filepath, + std::string_view entity_path_prefix = std::string_view(), bool timeless = false + ) const; /// Logs the given `contents` using all `DataLoader`s available. /// @@ -537,12 +570,22 @@ namespace rerun { /// \param filepath Path to the file that the `contents` belong to. /// \param contents Contents to be logged. /// \param contents_size Size in bytes of the `contents`. + /// \param entity_path_prefix What should the logged entity paths be prefixed with? + /// \param timeless Should the logged data be timeless? /// /// \see `try_log_file_from_contents` void log_file_from_contents( - const std::filesystem::path& filepath, const std::byte* contents, size_t contents_size + const std::filesystem::path& filepath, const std::byte* contents, size_t contents_size, + std::string_view entity_path_prefix = std::string_view(), bool timeless = false ) const { - try_log_file_from_contents(filepath, contents, contents_size).handle(); + try_log_file_from_contents( + filepath, + contents, + contents_size, + entity_path_prefix, + timeless + ) + .handle(); } /// Logs the given `contents` using all `DataLoader`s available. @@ -557,10 +600,14 @@ namespace rerun { /// \param filepath Path to the file that the `contents` belong to. /// \param contents Contents to be logged. /// \param contents_size Size in bytes of the `contents`. + /// \param recording_id The recommended `RecordingId` to log the data to. + /// \param entity_path_prefix What should the logged entity paths be prefixed with? + /// \param timeless Should the logged data be timeless? /// /// \see `log_file_from_contents` Error try_log_file_from_contents( - const std::filesystem::path& filepath, const std::byte* contents, size_t contents_size + const std::filesystem::path& filepath, const std::byte* contents, size_t contents_size, + std::string_view entity_path_prefix = std::string_view(), bool timeless = false ) const; /// @} From d0b5d8bbf7a1ebd8d281884295f448d4cc40a610 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Mon, 4 Mar 2024 15:52:43 +0100 Subject: [PATCH 2/5] update cpp examples --- examples/cpp/external_data_loader/main.cpp | 134 +++++++++++++-------- examples/cpp/log_file/main.cpp | 5 +- 2 files changed, 90 insertions(+), 49 deletions(-) diff --git a/examples/cpp/external_data_loader/main.cpp b/examples/cpp/external_data_loader/main.cpp index adddc44269c2..286e4d9f739c 100644 --- a/examples/cpp/external_data_loader/main.cpp +++ b/examples/cpp/external_data_loader/main.cpp @@ -1,11 +1,55 @@ +#include #include #include +#include #include #include #include +#include +#include + +void set_time_from_args(const rerun::RecordingStream& rec, cxxopts::ParseResult& args) { + if (args.count("time")) { + const auto times = args["time"].as>(); + for (const auto& time_str : times) { + auto pos = time_str.find('='); + if (pos != std::string::npos) { + auto timeline_name = time_str.substr(0, pos); + int64_t time = std::stol(time_str.substr(pos + 1)); + rec.set_time_seconds(timeline_name, time); + } + } + } + + if (args.count("sequence")) { + const auto sequences = args["sequence"].as>(); + for (const auto& sequence_str : sequences) { + auto pos = sequence_str.find('='); + if (pos != std::string::npos) { + auto timeline_name = sequence_str.substr(0, pos); + int64_t sequence = std::stol(sequence_str.substr(pos + 1)); + rec.set_time_sequence(timeline_name, sequence); + } + } + } +} -static const char* USAGE = R"( +int main(int argc, char* argv[]) { + // The Rerun Viewer will always pass these two pieces of information: + // 1. The path to be loaded, as a positional arg. + // 2. A shared recording ID, via the `--recording-id` flag. + // + // It is up to you whether you make use of that shared recording ID or not. + // If you use it, the data will end up in the same recording as all other plugins interested in + // that file, otherwise you can just create a dedicated recording for it. Or both. + // + // Check out `re_data_source::DataLoaderSettings` documentation for an exhaustive listing of + // the available CLI parameters. + + cxxopts::Options options( + "rerun-loader-cpp-file", + R"( This is an example executable data-loader plugin for the Rerun Viewer. Any executable on your `$PATH` with a name that starts with `rerun-loader-` will be treated as an external data-loader. @@ -15,57 +59,38 @@ special exit code to indicate that it doesn't support anything else. To try it out, compile it and place it in your $PATH as `rerun-loader-cpp-file`, then open a C++ source file with Rerun (`rerun file.cpp`). +)" + ); -USAGE: - rerun-loader-cpp-file [OPTIONS] FILEPATH - -FLAGS: - -h, --help Prints help information - -OPTIONS: - --recording-id RECORDING_ID ID of the shared recording - -ARGS: - -)"; - -int main(int argc, char* argv[]) { - // The Rerun Viewer will always pass these two pieces of information: - // 1. The path to be loaded, as a positional arg. - // 2. A shared recording ID, via the `--recording-id` flag. - // - // It is up to you whether you make use of that shared recording ID or not. - // If you use it, the data will end up in the same recording as all other plugins interested in - // that file, otherwise you can just create a dedicated recording for it. Or both. - std::string filepath; - std::string recording_id; - - for (int i = 1; i < argc; ++i) { - std::string arg(argv[i]); - - if (arg == "--recording-id") { - if (i + 1 < argc) { - recording_id = argv[i + 1]; - ++i; - } else { - std::cerr << USAGE << std::endl; - return 1; - } - } else { - filepath = arg; - } + // clang-format off + options.add_options() + ("h,help", "Print usage") + ("filepath", "The filepath to be loaded and logged", cxxopts::value()) + ("application-id", "Optional recommended ID for the application", cxxopts::value()) + ("recording-id", "Optional recommended ID for the recording", cxxopts::value()) + ("entity-path-prefix", "Optional prefix for all entity paths", cxxopts::value()) + ("timeless", "Optionally mark data to be logged as timeless", cxxopts::value()->default_value("false")) + ("time", "Optional timestamps to log at (e.g. `--time sim_time=1709203426`) (repeatable)", cxxopts::value>()) + ("sequence", "Optional sequences to log at (e.g. `--sequence sim_frame=42`) (repeatable)", cxxopts::value>()) + ; + // clang-format on + + options.parse_positional({"filepath"}); + + auto args = options.parse(argc, argv); + + if (args.count("help")) { + std::cout << options.help() << std::endl; + exit(0); } - if (filepath.empty()) { - std::cerr << USAGE << std::endl; - return 1; - } + const auto filepath = args["filepath"].as(); bool is_file = std::filesystem::is_regular_file(filepath); bool is_cpp_file = std::filesystem::path(filepath).extension().string() == ".cpp"; // Inform the Rerun Viewer that we do not support that kind of file. - if (!is_file || is_cpp_file) { + if (!is_file || !is_cpp_file) { return rerun::EXTERNAL_DATA_LOADER_INCOMPATIBLE_EXIT_CODE; } @@ -75,12 +100,27 @@ int main(int argc, char* argv[]) { std::string text = "## Some C++ code\n```cpp\n" + body.str() + "\n```\n"; - const auto rec = rerun::RecordingStream("rerun_example_external_data_loader", recording_id); + auto application_id = std::string_view("rerun_example_external_data_loader"); + if (args.count("application-id")) { + application_id = args["application-id"].as(); + } + auto recording_id = std::string_view(); + if (args.count("recording-id")) { + recording_id = args["recording-id"].as(); + } + const auto rec = rerun::RecordingStream(application_id, recording_id); // The most important part of this: log to standard output so the Rerun Viewer can ingest it! rec.to_stdout().exit_on_failure(); - rec.log_timeless( - filepath, + set_time_from_args(rec, args); + + auto entity_path = std::string(filepath); + if (args.count("entity-path-prefix")) { + entity_path = args["entity-path-prefix"].as() + "/" + filepath; + } + rec.log_with_timeless( + entity_path, + args["timeless"].as(), rerun::TextDocument(text).with_media_type(rerun::MediaType::markdown()) ); } diff --git a/examples/cpp/log_file/main.cpp b/examples/cpp/log_file/main.cpp index fa37f2e0aaa6..a1a6871bd78f 100644 --- a/examples/cpp/log_file/main.cpp +++ b/examples/cpp/log_file/main.cpp @@ -58,7 +58,7 @@ int main(int argc, char** argv) { for (const auto& filepath : filepaths) { if (!from_contents) { // Either log the file using its path… - rec.log_file_from_path(filepath); + rec.log_file_from_path(filepath, "log_file_example"); } else { // …or using its contents if you already have them loaded for some reason. if (std::filesystem::is_regular_file(filepath)) { @@ -70,7 +70,8 @@ int main(int argc, char** argv) { rec.log_file_from_contents( filepath, reinterpret_cast(data.c_str()), - data.size() + data.size(), + "log_file_example" ); } } From 2ad6d04ae4d496a3e032047996f1163bcf722001 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Mon, 4 Mar 2024 15:59:06 +0100 Subject: [PATCH 3/5] lints --- rerun_cpp/src/rerun/recording_stream.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/rerun_cpp/src/rerun/recording_stream.hpp b/rerun_cpp/src/rerun/recording_stream.hpp index 90b123c35093..57af84951830 100644 --- a/rerun_cpp/src/rerun/recording_stream.hpp +++ b/rerun_cpp/src/rerun/recording_stream.hpp @@ -527,7 +527,6 @@ namespace rerun { /// See for more information. /// /// \param filepath Path to the file to be logged. - /// \param recording_id The recommended `RecordingId` to log the data to. /// \param entity_path_prefix What should the logged entity paths be prefixed with? /// \param timeless Should the logged data be timeless? /// @@ -600,7 +599,6 @@ namespace rerun { /// \param filepath Path to the file that the `contents` belong to. /// \param contents Contents to be logged. /// \param contents_size Size in bytes of the `contents`. - /// \param recording_id The recommended `RecordingId` to log the data to. /// \param entity_path_prefix What should the logged entity paths be prefixed with? /// \param timeless Should the logged data be timeless? /// From 51456e5d3992bd62eaeb59953ede2458e040b95d Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Mon, 4 Mar 2024 16:15:00 +0100 Subject: [PATCH 4/5] lint --- rerun_cpp/src/rerun/recording_stream.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/rerun_cpp/src/rerun/recording_stream.hpp b/rerun_cpp/src/rerun/recording_stream.hpp index 57af84951830..3c1d204c9a6e 100644 --- a/rerun_cpp/src/rerun/recording_stream.hpp +++ b/rerun_cpp/src/rerun/recording_stream.hpp @@ -414,7 +414,6 @@ namespace rerun { /// Additional timelines set by `set_time_sequence` or `set_time` will also be included. /// \param archetypes_or_collectiones Any type for which the `AsComponents` trait is implemented. /// This is the case for any archetype or `std::vector`/`std::array`/C-array of components implements. - /// \returns An error if an error occurs during serialization or logging. /// /// @see log, try_log, log_timeless, try_log_timeless template From 25b281a0bb6c0e47015b339a8656c4056406efe9 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Mon, 4 Mar 2024 16:25:55 +0100 Subject: [PATCH 5/5] more lints, because every single platform must of course fail differently --- examples/cpp/external_data_loader/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cpp/external_data_loader/main.cpp b/examples/cpp/external_data_loader/main.cpp index 286e4d9f739c..0a39a9499db9 100644 --- a/examples/cpp/external_data_loader/main.cpp +++ b/examples/cpp/external_data_loader/main.cpp @@ -17,7 +17,7 @@ void set_time_from_args(const rerun::RecordingStream& rec, cxxopts::ParseResult& if (pos != std::string::npos) { auto timeline_name = time_str.substr(0, pos); int64_t time = std::stol(time_str.substr(pos + 1)); - rec.set_time_seconds(timeline_name, time); + rec.set_time_seconds(timeline_name, static_cast(time)); } } }