Skip to content

Commit

Permalink
Allow showing image shaped tensors in the tensor view (#3583)
Browse files Browse the repository at this point in the history
### What

* Fixes #3569

Testcode:
```
import numpy as np
import rerun as rr

rr.init("tensor", spawn=True)
rr.log("tensor", rr.Tensor(np.asarray(np.random.rand(100, 100, 3) * 255, dtype=np.uint8)))
rr.log("image", rr.Image(np.random.rand(100, 100, 3)))
```

Before the `tensor` entity wouldn't show anything, now this shows two
space views:
<img width="1737" alt="image"
src="https://github.com/rerun-io/rerun/assets/1220815/4542e132-b54e-4a2f-a722-a99fac262830">


In fact, we didn't allow tensor views for anything image shaped _at all_
before!

Tested with `python ./scripts/run_all.py --fast` and (todo!) webdemo
that this didn't break heuristic

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested [demo.rerun.io](https://demo.rerun.io/pr/3583) (if
applicable)

- [PR Build Summary](https://build.rerun.io/pr/3583)
- [Docs
preview](https://rerun.io/preview/51350829da593ce510e685754a5df4df4a3fb4fe/docs)
<!--DOCS-PREVIEW-->
- [Examples
preview](https://rerun.io/preview/51350829da593ce510e685754a5df4df4a3fb4fe/examples)
<!--EXAMPLES-PREVIEW-->
- [Recent benchmark results](https://ref.rerun.io/dev/bench/)
- [Wasm size tracking](https://ref.rerun.io/dev/sizes/)
  • Loading branch information
Wumpf authored Oct 2, 2023
1 parent 6cf7738 commit 02a0eae
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 54 deletions.
54 changes: 14 additions & 40 deletions crates/re_space_view_tensor/src/view_part_system.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ use re_types::{
Archetype, ComponentNameSet,
};
use re_viewer_context::{
default_heuristic_filter, NamedViewSystem, SpaceViewSystemExecutionError, TensorDecodeCache,
ViewContextCollection, ViewPartSystem, ViewQuery, ViewerContext,
NamedViewSystem, SpaceViewSystemExecutionError, TensorDecodeCache, ViewContextCollection,
ViewPartSystem, ViewQuery, ViewerContext,
};

#[derive(Default)]
Expand All @@ -35,28 +35,6 @@ impl ViewPartSystem for TensorSystem {
std::iter::once(Tensor::indicator().name()).collect()
}

/// Tensor view doesn't handle 2D images, see [`TensorSystem::load_tensor_entity`]
fn heuristic_filter(
&self,
store: &re_arrow_store::DataStore,
ent_path: &EntityPath,
query: &LatestAtQuery,
entity_components: &ComponentNameSet,
) -> bool {
if !default_heuristic_filter(entity_components, &self.indicator_components()) {
return false;
}

// NOTE: We want to make sure we query at the right time, otherwise we might take into
// account a `Clear()` that actually only applies into the future, and then
// `is_shaped_like_an_image` will righfully fail because of the empty tensor.
if let Some(tensor) = store.query_latest_component::<TensorData>(ent_path, query) {
!tensor.is_shaped_like_an_image() && !tensor.is_vector()
} else {
false
}
}

fn execute(
&mut self,
ctx: &mut ViewerContext<'_>,
Expand Down Expand Up @@ -93,22 +71,18 @@ impl TensorSystem {
_props: &EntityProperties,
tensor: TensorData,
) {
if !tensor.is_shaped_like_an_image() {
// NOTE: Tensors don't support batches at the moment so always splat.
let tensor_path_hash = InstancePathHash::entity_splat(ent_path).versioned(row_id);
match ctx
.cache
.entry(|c: &mut TensorDecodeCache| c.entry(tensor_path_hash, tensor.0))
{
Ok(tensor) => {
let instance_path = InstancePath::instance(ent_path.clone(), InstanceKey(0));
self.tensors.insert(instance_path, (row_id, tensor));
}
Err(err) => {
re_log::warn_once!(
"Failed to decode decoding tensor at path {ent_path}: {err}"
);
}
// NOTE: Tensors don't support batches at the moment so always splat.
let tensor_path_hash = InstancePathHash::entity_splat(ent_path).versioned(row_id);
match ctx
.cache
.entry(|c: &mut TensorDecodeCache| c.entry(tensor_path_hash, tensor.0))
{
Ok(tensor) => {
let instance_path = InstancePath::instance(ent_path.clone(), InstanceKey(0));
self.tensors.insert(instance_path, (row_id, tensor));
}
Err(err) => {
re_log::warn_once!("Failed to decode decoding tensor at path {ent_path}: {err}");
}
}
}
Expand Down
24 changes: 10 additions & 14 deletions crates/re_viewport/src/space_view_heuristics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@ use nohash_hasher::{IntMap, IntSet};

use re_arrow_store::{LatestAtQuery, Timeline};
use re_data_store::EntityPath;
use re_types::archetypes::Image;
use re_types::components::{DisconnectedSpace, TensorData};
use re_types::ComponentNameSet;
use re_types::{Archetype, ComponentNameSet};
use re_viewer_context::{
AutoSpawnHeuristic, SpaceViewClassName, ViewContextCollection, ViewPartCollection,
ViewSystemName, ViewerContext,
Expand Down Expand Up @@ -123,22 +124,17 @@ pub fn all_possible_space_views(
.collect_vec()
}

fn contains_any_image(
entity_path: &EntityPath,
store: &re_arrow_store::DataStore,
query: &LatestAtQuery,
) -> bool {
if let Some(tensor) = store.query_latest_component::<TensorData>(entity_path, query) {
tensor.is_shaped_like_an_image()
} else {
false
}
fn contains_any_image(ent_path: &EntityPath, store: &re_arrow_store::DataStore) -> bool {
store
.all_components(&Timeline::log_time(), ent_path)
.unwrap_or_default()
.iter()
.any(|comp| *comp == Image::indicator().name())
}

fn is_interesting_space_view_at_root(
data_store: &re_arrow_store::DataStore,
candidate: &SpaceViewBlueprint,
query: &LatestAtQuery,
) -> bool {
// Not interesting if it has only data blueprint groups and no direct entities.
// -> If there In that case we want spaceviews at those groups.
Expand All @@ -149,7 +145,7 @@ fn is_interesting_space_view_at_root(
// If there are any images directly under the root, don't create root space either.
// -> For images we want more fine grained control and resort to child-of-root spaces only.
for entity_path in &candidate.contents.root_group().entities {
if contains_any_image(entity_path, data_store, query) {
if contains_any_image(entity_path, data_store) {
return false;
}
}
Expand Down Expand Up @@ -208,7 +204,7 @@ pub fn default_created_space_views(
.iter()
.filter_map(|space_view_candidate| {
(space_view_candidate.space_origin.is_root()
&& is_interesting_space_view_at_root(store, space_view_candidate, &query))
&& is_interesting_space_view_at_root(store, space_view_candidate))
.then_some(*space_view_candidate.class_name())
})
.collect::<Vec<_>>();
Expand Down

0 comments on commit 02a0eae

Please sign in to comment.