Skip to content

Commit

Permalink
Extract panda3d renderer (#1921)
Browse files Browse the repository at this point in the history
* Add interface to start extraction

* Move renderer out

* Update smarts/core/renderer_base.py

Co-authored-by: Saul Field <[email protected]>

* Fix renderer inheritance.

* Finish extraction

* Update tests.

* Update changelog.

* Fix typing issue.

---------

Co-authored-by: Saul Field <[email protected]>
  • Loading branch information
Gamenot and saulfield authored Apr 25, 2023
1 parent 6d35ed9 commit f621ba1
Show file tree
Hide file tree
Showing 16 changed files with 463 additions and 176 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ Copy and pasting the git commit messages is __NOT__ enough.
- Benchmark listing may specify specialised metric formula for each benchmark.
- Changed `benchmark_runner_v0.py` to only average records across scenarios that share the same environment. Records are not averaged across different environments, because the scoring formula may differ in different environments.
- Renamed GapBetweenVehicles cost to VehicleGap cost in metric module.
- Camera metadata now uses radians instead of degrees.
- The `Panda3d` implementation of `Renderer` has been extracted from the interface and moved to `smarts.p3d`.
### Deprecated
### Fixed
- Fixed issues related to waypoints in junctions on Argoverse maps. Waypoints will now be generated for all paths leading through the lane(s) the vehicle is on.
Expand Down
2 changes: 1 addition & 1 deletion smarts/core/observations.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ class GridMapMetadata(NamedTuple):
"""Map height in # of cells."""
camera_position: Tuple[float, float, float]
"""Camera position when projected onto the map."""
camera_heading_in_degrees: float
camera_heading: float
"""Camera rotation angle along z-axis when projected onto the map."""


Expand Down
176 changes: 176 additions & 0 deletions smarts/core/renderer_base.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.


# to allow for typing to refer to class being defined (Renderer)
from __future__ import annotations

import logging
from dataclasses import dataclass
from enum import IntEnum
from typing import Tuple

import numpy as np

from .coordinates import Pose


class DEBUG_MODE(IntEnum):
"""The rendering debug information level."""

SPAM = 1
DEBUG = 2
INFO = 3
WARNING = 4
ERROR = 5


@dataclass
class OffscreenCamera:
"""A camera used for rendering images to a graphics buffer."""

renderer: RendererBase

def wait_for_ram_image(self, img_format: str, retries=100):
"""Attempt to acquire a graphics buffer."""
# Rarely, we see dropped frames where an image is not available
# for our observation calculations.
#
# We've seen this happen fairly reliable when we are initializing
# a multi-agent + multi-instance simulation.
#
# To deal with this, we can try to force a render and block until
# we are fairly certain we have an image in ram to return to the user
raise NotImplementedError

def update(self, pose: Pose, height: float):
"""Update the location of the camera.
Args:
pose:
The pose of the camera target.
height:
The height of the camera above the camera target.
"""
raise NotImplementedError

@property
def image_dimensions(self) -> Tuple[int, int]:
"""The dimensions of the output camera image."""
raise NotImplementedError

@property
def position(self) -> Tuple[float, float, float]:
"""The position of the camera."""
raise NotImplementedError

@property
def heading(self) -> float:
"""The heading of this camera."""
raise NotImplementedError

def teardown(self):
"""Clean up internal resources."""
raise NotImplementedError


class RendererBase:
"""The base class for rendering
Returns:
RendererBase:
"""

@property
def id(self):
"""The id of the simulation rendered."""
raise NotImplementedError

@property
def is_setup(self) -> bool:
"""If the renderer has been fully initialized."""
raise NotImplementedError

@property
def log(self) -> logging.Logger:
"""The rendering logger."""
raise NotImplementedError

def remove_buffer(self, buffer):
"""Remove the rendering buffer."""
raise NotImplementedError

def setup(self, scenario):
"""Initialize this renderer."""
raise NotImplementedError

def render(self):
"""Render the scene graph of the simulation."""
raise NotImplementedError

def reset(self):
"""Reset the render back to initialized state."""
raise NotImplementedError

def step(self):
"""provided for non-SMARTS uses; normally not used by SMARTS."""
raise NotImplementedError

def sync(self, sim_frame):
"""Update the current state of the vehicles within the renderer."""
raise NotImplementedError

def teardown(self):
"""Clean up internal resources."""
raise NotImplementedError

def destroy(self):
"""Destroy the renderer. Cleans up all remaining renderer resources."""
raise NotImplementedError

def create_vehicle_node(self, glb_model: str, vid: str, color, pose: Pose):
"""Create a vehicle node."""
raise NotImplementedError

def begin_rendering_vehicle(self, vid: str, is_agent: bool):
"""Add the vehicle node to the scene graph"""
raise NotImplementedError

def update_vehicle_node(self, vid: str, pose: Pose):
"""Move the specified vehicle node."""
raise NotImplementedError

def remove_vehicle_node(self, vid: str):
"""Remove a vehicle node"""
raise NotImplementedError

def camera_for_id(self, camera_id) -> OffscreenCamera:
"""Get a camera by its id."""
raise NotImplementedError

def build_offscreen_camera(
self,
name: str,
mask: int,
width: int,
height: int,
resolution: float,
) -> OffscreenCamera:
"""Generates a new offscreen camera."""
raise NotImplementedError
54 changes: 29 additions & 25 deletions smarts/core/sensor.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
Vias,
)
from smarts.core.plan import Plan
from smarts.core.renderer_base import RendererBase
from smarts.core.road_map import RoadMap, Waypoint
from smarts.core.signals import SignalState
from smarts.core.utils.math import squared_dist
Expand Down Expand Up @@ -79,8 +80,8 @@ class CameraSensor(Sensor):

def __init__(
self,
vehicle_state,
renderer, # type Renderer or None
vehicle_state: VehicleState,
renderer: RendererBase,
name: str,
mask: int,
width: int,
Expand Down Expand Up @@ -108,7 +109,7 @@ def __eq__(self, __value: object) -> bool:
)

def teardown(self, **kwargs):
renderer = kwargs.get("renderer")
renderer: Optional[RendererBase] = kwargs.get("renderer")
if not renderer:
return
camera = renderer.camera_for_id(self._camera_name)
Expand All @@ -121,7 +122,7 @@ def step(self, sim_frame, **kwargs):
sim_frame.actor_states_by_id[self._target_actor], kwargs.get("renderer")
)

def _follow_actor(self, vehicle_state, renderer):
def _follow_actor(self, vehicle_state: VehicleState, renderer: RendererBase):
if not renderer:
return
camera = renderer.camera_for_id(self._camera_name)
Expand All @@ -137,11 +138,11 @@ class DrivableAreaGridMapSensor(CameraSensor):

def __init__(
self,
vehicle_state,
vehicle_state: VehicleState,
width: int,
height: int,
resolution: float,
renderer, # type Renderer or None
renderer: RendererBase,
):
super().__init__(
vehicle_state,
Expand All @@ -154,22 +155,23 @@ def __init__(
)
self._resolution = resolution

def __call__(self, renderer) -> DrivableAreaGridMap:
def __call__(self, renderer: RendererBase) -> DrivableAreaGridMap:
camera = renderer.camera_for_id(self._camera_name)
assert camera is not None, "Drivable area grid map has not been initialized"

ram_image = camera.wait_for_ram_image(img_format="A")
mem_view = memoryview(ram_image)
image = np.frombuffer(mem_view, np.uint8)
image.shape = (camera.tex.getYSize(), camera.tex.getXSize(), 1)
image: np.ndarray = np.frombuffer(mem_view, np.uint8)
width, height = camera.image_dimensions
image.shape = (height, width, 1)
image = np.flipud(image)

metadata = GridMapMetadata(
resolution=self._resolution,
height=image.shape[0],
width=image.shape[1],
camera_position=camera.camera_np.getPos(),
camera_heading_in_degrees=camera.camera_np.getH(),
camera_position=camera.position,
camera_heading=camera.heading,
)
return DrivableAreaGridMap(data=image, metadata=metadata)

Expand All @@ -179,11 +181,11 @@ class OGMSensor(CameraSensor):

def __init__(
self,
vehicle_state,
vehicle_state: VehicleState,
width: int,
height: int,
resolution: float,
renderer, # type Renderer or None
renderer: RendererBase,
):
super().__init__(
vehicle_state,
Expand All @@ -196,22 +198,23 @@ def __init__(
)
self._resolution = resolution

def __call__(self, renderer) -> OccupancyGridMap:
def __call__(self, renderer: RendererBase) -> OccupancyGridMap:
camera = renderer.camera_for_id(self._camera_name)
assert camera is not None, "OGM has not been initialized"

ram_image = camera.wait_for_ram_image(img_format="A")
mem_view = memoryview(ram_image)
grid = np.frombuffer(mem_view, np.uint8)
grid.shape = (camera.tex.getYSize(), camera.tex.getXSize(), 1)
grid: np.ndarray = np.frombuffer(mem_view, np.uint8)
width, height = camera.image_dimensions
grid.shape = (height, width, 1)
grid = np.flipud(grid)

metadata = GridMapMetadata(
resolution=self._resolution,
height=grid.shape[0],
width=grid.shape[1],
camera_position=camera.camera_np.getPos(),
camera_heading_in_degrees=camera.camera_np.getH(),
camera_position=camera.position,
camera_heading=camera.heading,
)
return OccupancyGridMap(data=grid, metadata=metadata)

Expand All @@ -221,11 +224,11 @@ class RGBSensor(CameraSensor):

def __init__(
self,
vehicle_state,
vehicle_state: VehicleState,
width: int,
height: int,
resolution: float,
renderer, # type Renderer or None
renderer: RendererBase,
):
super().__init__(
vehicle_state,
Expand All @@ -238,22 +241,23 @@ def __init__(
)
self._resolution = resolution

def __call__(self, renderer) -> TopDownRGB:
def __call__(self, renderer: RendererBase) -> TopDownRGB:
camera = renderer.camera_for_id(self._camera_name)
assert camera is not None, "RGB has not been initialized"

ram_image = camera.wait_for_ram_image(img_format="RGB")
mem_view = memoryview(ram_image)
image = np.frombuffer(mem_view, np.uint8)
image.shape = (camera.tex.getYSize(), camera.tex.getXSize(), 3)
image: np.ndarray = np.frombuffer(mem_view, np.uint8)
width, height = camera.image_dimensions
image.shape = (height, width, 3)
image = np.flipud(image)

metadata = GridMapMetadata(
resolution=self._resolution,
height=image.shape[0],
width=image.shape[1],
camera_position=camera.camera_np.getPos(),
camera_heading_in_degrees=camera.camera_np.getH(),
camera_position=camera.position,
camera_heading=camera.heading,
)
return TopDownRGB(data=image, metadata=metadata)

Expand Down
5 changes: 3 additions & 2 deletions smarts/core/sensor_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from typing import Dict, FrozenSet, List, Optional, Set, Tuple

from smarts.core import config
from smarts.core.renderer_base import RendererBase
from smarts.core.sensors import Observation, Sensor, Sensors, SensorState
from smarts.core.sensors.local_sensor_resolver import LocalSensorResolver
from smarts.core.sensors.parallel_sensor_resolver import ParallelSensorResolver
Expand Down Expand Up @@ -85,7 +86,7 @@ def observe(
sim_frame,
sim_local_constants,
agent_ids,
renderer_ref,
renderer_ref: RendererBase,
physics_ref,
):
"""Runs observations and updates the sensor states.
Expand All @@ -96,7 +97,7 @@ def observe(
The values that should stay the same for a simulation over a reset.
agent_ids ({str, ...}):
The agent ids to process.
renderer_ref (Optional[Renderer]):
renderer_ref (Optional[RendererBase]):
The renderer (if any) that should be used.
physics_ref (bc.BulletClient):
The physics client.
Expand Down
Loading

0 comments on commit f621ba1

Please sign in to comment.