-
Notifications
You must be signed in to change notification settings - Fork 373
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
New data APIs 6: cached archetype queries (#5673)
Introduce very high-level APIs that make it possible to query, resolve,
deserialize and cache an entire archetype all at once.
The core trait allowing this is the newly introduced `ToArchetype<A>`
trait:
```rust
pub trait ToArchetype<A: re_types_core::Archetype> {
fn to_archetype(&self, resolver: &crate::PromiseResolver) -> crate::PromiseResult<A>;
}
```
This trait is implemented for all builtins archetypes thanks to a new
codegen pass.
Implementing such a trait is tricky: one needs to know about this trait,
archetypes, queries, caches, etc all in one single place. This is a
recipe for a very nasty circular dependency chains.
This PR makes it work for all archetypes except archetypes that are
generated in `re_viewport`. These will need some special care in a
follow-up PR (likely moving them out of there, and only reexporting them
in `re_viewport`?).
Update: related:
- #5421
Here's how it looks in practice:
```rust
let caches = re_query_cache2::Caches::new(&store);
// First, get the results for this query.
//
// They might or might not already be cached. We won't know for sure until we try to access
// each individual component's data below.
let results: CachedLatestAtResults = caches.latest_at(
&store,
&query,
&entity_path.into(),
Points2D::all_components().iter().cloned(), // no generics!
);
// Then make use of the `ToArchetype` helper trait in order to query, resolve, deserialize and
// cache an entire archetype all at once.
use re_query_cache2::ToArchetype as _;
let arch: Points2D = match results.to_archetype(&resolver) {
PromiseResult::Pending => {
// Handle the fact that the data isn't ready appropriately.
return Ok(());
}
PromiseResult::Ready(arch) => arch,
PromiseResult::Error(err) => return Err(err.into()),
};
// With the data now fully resolved/converted and deserialized, some joining logic can be
// applied if desired.
//
// In most cases this will be either a clamped zip, or no joining at all.
let color_default_fn = || None;
let label_default_fn = || None;
let results = clamped_zip_1x2(
arch.positions.iter(),
arch.colors
.iter()
.flat_map(|colors| colors.iter().map(Some)),
color_default_fn,
arch.labels
.iter()
.flat_map(|labels| labels.iter().map(Some)),
label_default_fn,
)
.collect_vec();
eprintln!("results:\n{results:?}");
```
---
Part of a PR series to completely revamp the data APIs in preparation
for the removal of instance keys and the introduction of promises:
- #5573
- #5574
- #5581
- #5605
- #5606
- #5633
- #5673
- #5679
- #5687
- #5755
- TODO
- TODO
Builds on top of the static data PR series:
- #5534- Loading branch information
Showing
46 changed files
with
2,795 additions
and
46 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| use itertools::Itertools; | ||
| use re_data_store::{DataStore, LatestAtQuery}; | ||
| use re_log_types::{build_frame_nr, DataRow, RowId, TimeType, Timeline}; | ||
| use re_types::{ | ||
| archetypes::Points2D, | ||
| components::{Color, Position2D, Text}, | ||
| }; | ||
| use re_types_core::{Archetype as _, Loggable as _}; | ||
|
|
||
| use re_query_cache2::{clamped_zip_1x2, CachedLatestAtResults, PromiseResolver, PromiseResult}; | ||
|
|
||
| // --- | ||
|
|
||
| fn main() -> anyhow::Result<()> { | ||
| let store = store()?; | ||
| eprintln!("store:\n{}", store.to_data_table()?); | ||
|
|
||
| let resolver = PromiseResolver::default(); | ||
|
|
||
| let entity_path = "points"; | ||
| let timeline = Timeline::new("frame_nr", TimeType::Sequence); | ||
| let query = LatestAtQuery::latest(timeline); | ||
| eprintln!("query:{query:?}"); | ||
|
|
||
| let caches = re_query_cache2::Caches::new(&store); | ||
|
|
||
| // First, get the results for this query. | ||
| // | ||
| // They might or might not already be cached. We won't know for sure until we try to access | ||
| // each individual component's data below. | ||
| let results: CachedLatestAtResults = caches.latest_at( | ||
| &store, | ||
| &query, | ||
| &entity_path.into(), | ||
| Points2D::all_components().iter().cloned(), // no generics! | ||
| ); | ||
|
|
||
| // Then make use of the `ToArchetype` helper trait in order to query, resolve, deserialize and | ||
| // cache an entire archetype all at once. | ||
| use re_query_cache2::ToArchetype as _; | ||
|
|
||
| let arch: Points2D = match results.to_archetype(&resolver).flatten() { | ||
| PromiseResult::Pending => { | ||
| // Handle the fact that the data isn't ready appropriately. | ||
| return Ok(()); | ||
| } | ||
| PromiseResult::Ready(arch) => arch, | ||
| PromiseResult::Error(err) => return Err(err.into()), | ||
| }; | ||
|
|
||
| // With the data now fully resolved/converted and deserialized, some joining logic can be | ||
| // applied if desired. | ||
| // | ||
| // In most cases this will be either a clamped zip, or no joining at all. | ||
|
|
||
| let color_default_fn = || None; | ||
| let label_default_fn = || None; | ||
|
|
||
| let results = clamped_zip_1x2( | ||
| arch.positions.iter(), | ||
| arch.colors | ||
| .iter() | ||
| .flat_map(|colors| colors.iter().map(Some)), | ||
| color_default_fn, | ||
| arch.labels | ||
| .iter() | ||
| .flat_map(|labels| labels.iter().map(Some)), | ||
| label_default_fn, | ||
| ) | ||
| .collect_vec(); | ||
|
|
||
| eprintln!("results:\n{results:?}"); | ||
|
|
||
| Ok(()) | ||
| } | ||
|
|
||
| // --- | ||
|
|
||
| fn store() -> anyhow::Result<DataStore> { | ||
| let mut store = DataStore::new( | ||
| re_log_types::StoreId::random(re_log_types::StoreKind::Recording), | ||
| re_types::components::InstanceKey::name(), | ||
| Default::default(), | ||
| ); | ||
|
|
||
| let entity_path = "points"; | ||
|
|
||
| { | ||
| let timepoint = [build_frame_nr(123)]; | ||
|
|
||
| let points = vec![Position2D::new(1.0, 2.0), Position2D::new(3.0, 4.0)]; | ||
| let row = DataRow::from_cells1_sized(RowId::new(), entity_path, timepoint, 2, points)?; | ||
| store.insert_row(&row)?; | ||
|
|
||
| let colors = vec![Color::from_rgb(255, 0, 0)]; | ||
| let row = DataRow::from_cells1_sized(RowId::new(), entity_path, timepoint, 1, colors)?; | ||
| store.insert_row(&row)?; | ||
|
|
||
| let labels = vec![Text("a".into()), Text("b".into())]; | ||
| let row = DataRow::from_cells1_sized(RowId::new(), entity_path, timepoint, 2, labels)?; | ||
| store.insert_row(&row)?; | ||
| } | ||
|
|
||
| Ok(store) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.