Skip to content

Commit

Permalink
Add blueprint to Gesture Detection example (#5619)
Browse files Browse the repository at this point in the history
### What

Allows getting rid of redundant logging \o/ and making all paths lower
case


![image](https://github.com/rerun-io/rerun/assets/1220815/eb710dbe-930c-4c9f-9499-33090bc4b191)

cc: @andreasnaoum 


### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using newly built examples:
[app.rerun.io](https://app.rerun.io/pr/5619/index.html)
* Using examples from latest `main` build:
[app.rerun.io](https://app.rerun.io/pr/5619/index.html?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[app.rerun.io](https://app.rerun.io/pr/5619/index.html?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)!

- [PR Build Summary](https://build.rerun.io/pr/5619)
- [Docs
preview](https://rerun.io/preview/b4c62bf6da4d6fa8e105ddb33a1d222ee2439ae8/docs)
<!--DOCS-PREVIEW-->
- [Examples
preview](https://rerun.io/preview/b4c62bf6da4d6fa8e105ddb33a1d222ee2439ae8/examples)
<!--EXAMPLES-PREVIEW-->
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)
  • Loading branch information
Wumpf authored Mar 21, 2024
1 parent 3844083 commit 0530f58
Showing 1 changed file with 25 additions and 12 deletions.
37 changes: 25 additions & 12 deletions examples/python/gesture_detection/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import numpy.typing as npt
import requests
import rerun as rr # pip install rerun-sdk
import rerun.blueprint as rrb
import tqdm
from mediapipe.tasks import python
from mediapipe.tasks.python import vision
Expand Down Expand Up @@ -87,8 +88,7 @@ def __init__(self, video_mode: bool = False):
),
timeless=True,
)
# rr.log("Hand3D", rr.ViewCoordinates.RIGHT_HAND_Y_DOWN, timeless=True)
rr.log("Hand3D", rr.ViewCoordinates.LEFT_HAND_Y_DOWN, timeless=True)
rr.log("hand3d", rr.ViewCoordinates.LEFT_HAND_Y_DOWN, timeless=True)

@staticmethod
def convert_landmarks_to_image_coordinates(
Expand All @@ -111,7 +111,7 @@ def detect_and_log(self, image: npt.NDArray[np.uint8], frame_time_nano: int) ->
else self.recognizer.recognize(image)
)

for log_key in ["Media/Points", "Hand/Points", "Media/Connections", "Hand/Connections", "Hand3D/Points"]:
for log_key in ["hand2d/points", "hand2d/connections", "hand3d/points"]:
rr.log(log_key, rr.Clear(recursive=True))

for i, gesture in enumerate(recognition_result.gestures):
Expand All @@ -125,7 +125,7 @@ def detect_and_log(self, image: npt.NDArray[np.uint8], frame_time_nano: int) ->
landmark_positions_3d = self.convert_landmarks_to_3d(hand_landmarks)
if landmark_positions_3d is not None:
rr.log(
"Hand3D/Points",
"hand3d/points",
rr.Points3D(
landmark_positions_3d,
radii=20,
Expand All @@ -138,17 +138,15 @@ def detect_and_log(self, image: npt.NDArray[np.uint8], frame_time_nano: int) ->
points = self.convert_landmarks_to_image_coordinates(hand_landmarks, width, height)

# Log points to the image and Hand Entity
for log_key in ["Media/Points", "Hand/Points"]:
rr.log(log_key, rr.Points2D(points, radii=10, colors=[255, 0, 0]))
rr.log("hand2d/points", rr.Points2D(points, radii=10, colors=[255, 0, 0]))

# Obtain hand connections from MediaPipe
mp_hands_connections = mp.solutions.hands.HAND_CONNECTIONS
points1 = [points[connection[0]] for connection in mp_hands_connections]
points2 = [points[connection[1]] for connection in mp_hands_connections]

# Log connections to the image and Hand Entity [128, 128, 128]
for log_key in ["Media/Connections", "Hand/Connections"]:
rr.log(log_key, rr.LineStrips2D(np.stack((points1, points2), axis=1), colors=[255, 165, 0]))
rr.log("hand2d/connections", rr.LineStrips2D(np.stack((points1, points2), axis=1), colors=[255, 165, 0]))

def present_detected_gesture(self, category: str) -> None:
# Get the corresponding ulr of the picture for the detected gesture category
Expand All @@ -159,7 +157,7 @@ def present_detected_gesture(self, category: str) -> None:

# Log the detection by using the appropriate image
rr.log(
"Detection",
"detection",
rr.TextDocument(f"![Image]({GESTURE_URL + gesture_pic})".strip(), media_type=rr.MediaType.MARKDOWN),
)

Expand Down Expand Up @@ -195,7 +193,7 @@ def run_from_sample_image(path: Path | str) -> None:
image = cv2.imread(str(path))
# image = resize_image(image, max_dim)
show_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
rr.log("Media/Image", rr.Image(show_image))
rr.log("media/image", rr.Image(show_image))
logger = GestureDetectorLogger(video_mode=False)
logger.detect_and_log(show_image, 0)

Expand Down Expand Up @@ -245,7 +243,7 @@ def run_from_video_capture(vid: int | str, max_frame_count: int | None) -> None:
rr.set_time_sequence("frame_nr", frame_idx)
rr.set_time_nanos("frame_time", frame_time_nano)
detector.detect_and_log(frame, frame_time_nano)
rr.log("Media/Video", rr.Image(frame).compress(jpeg_quality=75))
rr.log("media/video", rr.Image(frame).compress(jpeg_quality=75))

except KeyboardInterrupt:
pass
Expand Down Expand Up @@ -298,7 +296,22 @@ def main() -> None:
logging.warning(f"unknown arg: {arg}")

# Set up Rerun with script name
rr.script_setup(args, "rerun_example_mp_gesture_recognition")
rr.script_setup(
args,
"rerun_example_mp_gesture_recognition",
blueprint=rrb.Horizontal(
rrb.Spatial2DView(name="Input & Hand", contents=["media/**", "hand2d/**"]),
rrb.Vertical(
rrb.Tabs(
rrb.Spatial3DView(name="Hand 3D", origin="hand3d"),
rrb.Spatial2DView(name="Hand 2D", origin="hand2d"),
),
rrb.TextDocumentView(name="Detection", origin="detection"),
row_shares=[3, 2],
),
column_shares=[3, 1],
),
)

# Choose the appropriate run mode based on provided arguments
if args.demo_image:
Expand Down

0 comments on commit 0530f58

Please sign in to comment.