Skip to content

Commit

Permalink
Add blueprint to the nuscenes example (#5556)
Browse files Browse the repository at this point in the history
### What
* Part of #5468

An interesting part of this is that the blueprint is dynamically created
based on the the data (which cameras there are).

Before:

![image](https://github.com/rerun-io/rerun/assets/1148717/e8c8c5f2-6502-4325-8268-1067d824494a)


After:

![image](https://github.com/rerun-io/rerun/assets/1148717/60e0673e-5e72-4dc8-8bc3-d5c6e49c7d64)




The 3D camera is (still) too zoomed out, but gathering all the 2D images
in a tab view puts more focus on the 3D view.



### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using newly built examples:
[app.rerun.io](https://app.rerun.io/pr/5556/index.html)
* Using examples from latest `main` build:
[app.rerun.io](https://app.rerun.io/pr/5556/index.html?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[app.rerun.io](https://app.rerun.io/pr/5556/index.html?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)!

- [PR Build Summary](https://build.rerun.io/pr/5556)
- [Docs
preview](https://rerun.io/preview/b620da1066ab9c4ba09a3bd0dee813cc3af1de57/docs)
<!--DOCS-PREVIEW-->
- [Examples
preview](https://rerun.io/preview/b620da1066ab9c4ba09a3bd0dee813cc3af1de57/examples)
<!--EXAMPLES-PREVIEW-->
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)
  • Loading branch information
emilk authored Mar 18, 2024
1 parent cb19989 commit 71cd663
Show file tree
Hide file tree
Showing 5 changed files with 64 additions and 29 deletions.
2 changes: 1 addition & 1 deletion crates/re_viewer/src/ui/selection_panel.rs
Original file line number Diff line number Diff line change
Expand Up @@ -855,7 +855,7 @@ fn blueprint_ui_for_space_view(
.root_handle()
.and_then(|root| query_result.tree.lookup_result(root))
else {
re_log::error!("Could not find root data result for Space View {space_view_id:?}");
re_log::error_once!("Could not find root data result for Space View {space_view_id:?}");
return;
};

Expand Down
40 changes: 19 additions & 21 deletions examples/python/arkit_scenes/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -330,31 +330,29 @@ def main() -> None:

primary_camera_entity = highres_entity_path if args.include_highres else lowres_posed_entity_path

rr.script_setup(
args,
"rerun_example_arkit_scenes",
blueprint=rbl.Horizontal(
rbl.Spatial3DView(name="3D"),
rbl.Vertical(
rbl.Tabs(
# Note that we re-project the annotations into the 2D views:
# For this to work, the origin of the 2D views has to be a pinhole camera,
# this way the viewer knows how to project the 3D annotations into the 2D views.
rbl.Spatial2DView(
name="RGB",
origin=primary_camera_entity,
contents=[f"{primary_camera_entity}/rgb", "/world/annotations/**"],
),
rbl.Spatial2DView(
name="Depth",
origin=primary_camera_entity,
contents=[f"{primary_camera_entity}/depth", "/world/annotations/**"],
),
blueprint = rbl.Horizontal(
rbl.Spatial3DView(name="3D"),
rbl.Vertical(
rbl.Tabs(
# Note that we re-project the annotations into the 2D views:
# For this to work, the origin of the 2D views has to be a pinhole camera,
# this way the viewer knows how to project the 3D annotations into the 2D views.
rbl.Spatial2DView(
name="RGB",
origin=primary_camera_entity,
contents=["$origin/rgb", "/world/annotations/**"],
),
rbl.Spatial2DView(
name="Depth",
origin=primary_camera_entity,
contents=["$origin/depth", "/world/annotations/**"],
),
rbl.TextDocumentView(name="Readme"),
),
rbl.TextDocumentView(name="Readme"),
),
)

rr.script_setup(args, "rerun_example_arkit_scenes", blueprint=blueprint)
recording_path = ensure_recording_available(args.video_id, args.include_highres)
log_arkit(recording_path, args.include_highres)

Expand Down
45 changes: 41 additions & 4 deletions examples/python/nuscenes/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import matplotlib
import numpy as np
import rerun as rr
import rerun.blueprint as rbl
from download_dataset import MINISPLIT_SCENES, download_minisplit
from nuscenes import nuscenes

Expand Down Expand Up @@ -47,9 +48,28 @@ def ensure_scene_available(root_dir: pathlib.Path, dataset_version: str, scene_n
raise ValueError(f"{scene_name=} not found in dataset")


def log_nuscenes(root_dir: pathlib.Path, dataset_version: str, scene_name: str, max_time_sec: float) -> None:
def nuscene_sensor_names(nusc: nuscenes.NuScenes, scene_name: str) -> set[str]:
"""Return all sensor names in the scene."""

sensor_names = set()

scene = next(s for s in nusc.scene if s["name"] == scene_name)
first_sample = nusc.get("sample", scene["first_sample_token"])
for sample_data_token in first_sample["data"].values():
sample_data = nusc.get("sample_data", sample_data_token)
if sample_data["sensor_modality"] == "camera":
current_camera_token = sample_data_token
while current_camera_token != "":
sample_data = nusc.get("sample_data", current_camera_token)
sensor_name = sample_data["channel"]
sensor_names.add(sensor_name)
current_camera_token = sample_data["next"]

return sensor_names


def log_nuscenes(nusc: nuscenes.NuScenes, scene_name: str, max_time_sec: float) -> None:
"""Log nuScenes scene."""
nusc = nuscenes.NuScenes(version=dataset_version, dataroot=root_dir, verbose=True)

scene = next(s for s in nusc.scene if s["name"] == scene_name)

Expand Down Expand Up @@ -233,8 +253,25 @@ def main() -> None:

ensure_scene_available(args.root_dir, args.dataset_version, args.scene_name)

rr.script_setup(args, "rerun_example_nuscenes")
log_nuscenes(args.root_dir, args.dataset_version, args.scene_name, max_time_sec=args.seconds)
nusc = nuscenes.NuScenes(version=args.dataset_version, dataroot=args.root_dir, verbose=True)

# Set up the Rerun Blueprint (how the visualization is organized):
sensor_space_views = [
rbl.Spatial2DView(
name=sensor_name,
origin=f"world/ego_vehicle/{sensor_name}",
)
for sensor_name in nuscene_sensor_names(nusc, args.scene_name)
]
blueprint = rbl.Vertical(
rbl.Spatial3DView(name="3D", origin="world"),
rbl.Grid(*sensor_space_views),
row_shares=[3, 2],
)

rr.script_setup(args, "rerun_example_nuscenes", blueprint=blueprint)

log_nuscenes(nusc, args.scene_name, max_time_sec=args.seconds)

rr.script_teardown(args)

Expand Down
2 changes: 1 addition & 1 deletion rerun_py/rerun_sdk/rerun/blueprint/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ def __init__(
This is only applicable to `Horizontal` or `Grid` containers.
row_shares
The layout shares of the rows in the container. The share is used to determine what fraction of the total height each
row should take up. The ros with index `i` will take up the fraction `shares[i] / total_shares`.
row should take up. The row with index `i` will take up the fraction `shares[i] / total_shares`.
This is only applicable to `Vertical` or `Grid` containers.
grid_columns
The number of columns in the grid. This is only applicable to `Grid` containers.
Expand Down
4 changes: 2 additions & 2 deletions rerun_py/rerun_sdk/rerun/blueprint/containers.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ def __init__(self, *contents: Container | SpaceView, row_shares: Optional[RowSha
All positional arguments are the contents of the container, which may be either other containers or space views.
row_shares
The layout shares of the rows in the container. The share is used to determine what fraction of the total height each
row should take up. The ros with index `i` will take up the fraction `shares[i] / total_shares`.
row should take up. The row with index `i` will take up the fraction `shares[i] / total_shares`.
"""
super().__init__(*contents, kind=ContainerKind.Vertical, row_shares=row_shares)
Expand Down Expand Up @@ -67,7 +67,7 @@ def __init__(
column should take up. The column with index `i` will take up the fraction `shares[i] / total_shares`.
row_shares
The layout shares of the rows in the container. The share is used to determine what fraction of the total height each
row should take up. The ros with index `i` will take up the fraction `shares[i] / total_shares`.
row should take up. The row with index `i` will take up the fraction `shares[i] / total_shares`.
grid_columns
The number of columns in the grid.
Expand Down

0 comments on commit 71cd663

Please sign in to comment.