Extract panda3d renderer (#1921)

* Add interface to start extraction

* Move renderer out

* Update smarts/core/renderer_base.py

Co-authored-by: Saul Field <[email protected]>

* Fix renderer inheritance.

* Finish extraction

* Update tests.

* Update changelog.

* Fix typing issue.

---------

Co-authored-by: Saul Field <[email protected]>

Author: Gamenot

CHANGELOG.md

@@ -51,6 +51,8 @@ Copy and pasting the git commit messages is __NOT__ enough.
 - Benchmark listing may specify specialised metric formula for each benchmark.
 - Changed `benchmark_runner_v0.py` to only average records across scenarios that share the same environment. Records are not averaged across different environments, because the scoring formula may differ in different environments.
 - Renamed GapBetweenVehicles cost to VehicleGap cost in metric module.
+- Camera metadata now uses radians instead of degrees.
+- The `Panda3d` implementation of `Renderer` has been extracted from the interface and moved to `smarts.p3d`.
 ### Deprecated
 ### Fixed
 - Fixed issues related to waypoints in junctions on Argoverse maps. Waypoints will now be generated for all paths leading through the lane(s) the vehicle is on.

smarts/core/observations.py

@@ -115,7 +115,7 @@ class GridMapMetadata(NamedTuple):
     """Map height in # of cells."""
     camera_position: Tuple[float, float, float]
     """Camera position when projected onto the map."""
-    camera_heading_in_degrees: float
+    camera_heading: float
     """Camera rotation angle along z-axis when projected onto the map."""
 
 

smarts/core/renderer_base.py

@@ -0,0 +1,176 @@
+# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# to allow for typing to refer to class being defined (Renderer)
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from enum import IntEnum
+from typing import Tuple
+
+import numpy as np
+
+from .coordinates import Pose
+
+
+class DEBUG_MODE(IntEnum):
+    """The rendering debug information level."""
+
+    SPAM = 1
+    DEBUG = 2
+    INFO = 3
+    WARNING = 4
+    ERROR = 5
+
+
+@dataclass
+class OffscreenCamera:
+    """A camera used for rendering images to a graphics buffer."""
+
+    renderer: RendererBase
+
+    def wait_for_ram_image(self, img_format: str, retries=100):
+        """Attempt to acquire a graphics buffer."""
+        # Rarely, we see dropped frames where an image is not available
+        # for our observation calculations.
+        #
+        # We've seen this happen fairly reliable when we are initializing
+        # a multi-agent + multi-instance simulation.
+        #
+        # To deal with this, we can try to force a render and block until
+        # we are fairly certain we have an image in ram to return to the user
+        raise NotImplementedError
+
+    def update(self, pose: Pose, height: float):
+        """Update the location of the camera.
+        Args:
+            pose:
+                The pose of the camera target.
+            height:
+                The height of the camera above the camera target.
+        """
+        raise NotImplementedError
+
+    @property
+    def image_dimensions(self) -> Tuple[int, int]:
+        """The dimensions of the output camera image."""
+        raise NotImplementedError
+
+    @property
+    def position(self) -> Tuple[float, float, float]:
+        """The position of the camera."""
+        raise NotImplementedError
+
+    @property
+    def heading(self) -> float:
+        """The heading of this camera."""
+        raise NotImplementedError
+
+    def teardown(self):
+        """Clean up internal resources."""
+        raise NotImplementedError
+
+
+class RendererBase:
+    """The base class for rendering
+
+    Returns:
+        RendererBase:
+    """
+
+    @property
+    def id(self):
+        """The id of the simulation rendered."""
+        raise NotImplementedError
+
+    @property
+    def is_setup(self) -> bool:
+        """If the renderer has been fully initialized."""
+        raise NotImplementedError
+
+    @property
+    def log(self) -> logging.Logger:
+        """The rendering logger."""
+        raise NotImplementedError
+
+    def remove_buffer(self, buffer):
+        """Remove the rendering buffer."""
+        raise NotImplementedError
+
+    def setup(self, scenario):
+        """Initialize this renderer."""
+        raise NotImplementedError
+
+    def render(self):
+        """Render the scene graph of the simulation."""
+        raise NotImplementedError
+
+    def reset(self):
+        """Reset the render back to initialized state."""
+        raise NotImplementedError
+
+    def step(self):
+        """provided for non-SMARTS uses; normally not used by SMARTS."""
+        raise NotImplementedError
+
+    def sync(self, sim_frame):
+        """Update the current state of the vehicles within the renderer."""
+        raise NotImplementedError
+
+    def teardown(self):
+        """Clean up internal resources."""
+        raise NotImplementedError
+
+    def destroy(self):
+        """Destroy the renderer. Cleans up all remaining renderer resources."""
+        raise NotImplementedError
+
+    def create_vehicle_node(self, glb_model: str, vid: str, color, pose: Pose):
+        """Create a vehicle node."""
+        raise NotImplementedError
+
+    def begin_rendering_vehicle(self, vid: str, is_agent: bool):
+        """Add the vehicle node to the scene graph"""
+        raise NotImplementedError
+
+    def update_vehicle_node(self, vid: str, pose: Pose):
+        """Move the specified vehicle node."""
+        raise NotImplementedError
+
+    def remove_vehicle_node(self, vid: str):
+        """Remove a vehicle node"""
+        raise NotImplementedError
+
+    def camera_for_id(self, camera_id) -> OffscreenCamera:
+        """Get a camera by its id."""
+        raise NotImplementedError
+
+    def build_offscreen_camera(
+        self,
+        name: str,
+        mask: int,
+        width: int,
+        height: int,
+        resolution: float,
+    ) -> OffscreenCamera:
+        """Generates a new offscreen camera."""
+        raise NotImplementedError

smarts/core/sensor.py

@@ -46,6 +46,7 @@
     Vias,
 )
 from smarts.core.plan import Plan
+from smarts.core.renderer_base import RendererBase
 from smarts.core.road_map import RoadMap, Waypoint
 from smarts.core.signals import SignalState
 from smarts.core.utils.math import squared_dist
@@ -79,8 +80,8 @@ class CameraSensor(Sensor):
 
     def __init__(
         self,
-        vehicle_state,
-        renderer,  # type Renderer or None
+        vehicle_state: VehicleState,
+        renderer: RendererBase,
         name: str,
         mask: int,
         width: int,
@@ -108,7 +109,7 @@ def __eq__(self, __value: object) -> bool:
         )
 
     def teardown(self, **kwargs):
-        renderer = kwargs.get("renderer")
+        renderer: Optional[RendererBase] = kwargs.get("renderer")
         if not renderer:
             return
         camera = renderer.camera_for_id(self._camera_name)
@@ -121,7 +122,7 @@ def step(self, sim_frame, **kwargs):
             sim_frame.actor_states_by_id[self._target_actor], kwargs.get("renderer")
         )
 
-    def _follow_actor(self, vehicle_state, renderer):
+    def _follow_actor(self, vehicle_state: VehicleState, renderer: RendererBase):
         if not renderer:
             return
         camera = renderer.camera_for_id(self._camera_name)
@@ -137,11 +138,11 @@ class DrivableAreaGridMapSensor(CameraSensor):
 
     def __init__(
         self,
-        vehicle_state,
+        vehicle_state: VehicleState,
         width: int,
         height: int,
         resolution: float,
-        renderer,  # type Renderer or None
+        renderer: RendererBase,
     ):
         super().__init__(
             vehicle_state,
@@ -154,22 +155,23 @@ def __init__(
         )
         self._resolution = resolution
 
-    def __call__(self, renderer) -> DrivableAreaGridMap:
+    def __call__(self, renderer: RendererBase) -> DrivableAreaGridMap:
         camera = renderer.camera_for_id(self._camera_name)
         assert camera is not None, "Drivable area grid map has not been initialized"
 
         ram_image = camera.wait_for_ram_image(img_format="A")
         mem_view = memoryview(ram_image)
-        image = np.frombuffer(mem_view, np.uint8)
-        image.shape = (camera.tex.getYSize(), camera.tex.getXSize(), 1)
+        image: np.ndarray = np.frombuffer(mem_view, np.uint8)
+        width, height = camera.image_dimensions
+        image.shape = (height, width, 1)
         image = np.flipud(image)
 
         metadata = GridMapMetadata(
             resolution=self._resolution,
             height=image.shape[0],
             width=image.shape[1],
-            camera_position=camera.camera_np.getPos(),
-            camera_heading_in_degrees=camera.camera_np.getH(),
+            camera_position=camera.position,
+            camera_heading=camera.heading,
         )
         return DrivableAreaGridMap(data=image, metadata=metadata)
 
@@ -179,11 +181,11 @@ class OGMSensor(CameraSensor):
 
     def __init__(
         self,
-        vehicle_state,
+        vehicle_state: VehicleState,
         width: int,
         height: int,
         resolution: float,
-        renderer,  # type Renderer or None
+        renderer: RendererBase,
     ):
         super().__init__(
             vehicle_state,
@@ -196,22 +198,23 @@ def __init__(
         )
         self._resolution = resolution
 
-    def __call__(self, renderer) -> OccupancyGridMap:
+    def __call__(self, renderer: RendererBase) -> OccupancyGridMap:
         camera = renderer.camera_for_id(self._camera_name)
         assert camera is not None, "OGM has not been initialized"
 
         ram_image = camera.wait_for_ram_image(img_format="A")
         mem_view = memoryview(ram_image)
-        grid = np.frombuffer(mem_view, np.uint8)
-        grid.shape = (camera.tex.getYSize(), camera.tex.getXSize(), 1)
+        grid: np.ndarray = np.frombuffer(mem_view, np.uint8)
+        width, height = camera.image_dimensions
+        grid.shape = (height, width, 1)
         grid = np.flipud(grid)
 
         metadata = GridMapMetadata(
             resolution=self._resolution,
             height=grid.shape[0],
             width=grid.shape[1],
-            camera_position=camera.camera_np.getPos(),
-            camera_heading_in_degrees=camera.camera_np.getH(),
+            camera_position=camera.position,
+            camera_heading=camera.heading,
         )
         return OccupancyGridMap(data=grid, metadata=metadata)
 
@@ -221,11 +224,11 @@ class RGBSensor(CameraSensor):
 
     def __init__(
         self,
-        vehicle_state,
+        vehicle_state: VehicleState,
         width: int,
         height: int,
         resolution: float,
-        renderer,  # type Renderer or None
+        renderer: RendererBase,
     ):
         super().__init__(
             vehicle_state,
@@ -238,22 +241,23 @@ def __init__(
         )
         self._resolution = resolution
 
-    def __call__(self, renderer) -> TopDownRGB:
+    def __call__(self, renderer: RendererBase) -> TopDownRGB:
         camera = renderer.camera_for_id(self._camera_name)
         assert camera is not None, "RGB has not been initialized"
 
         ram_image = camera.wait_for_ram_image(img_format="RGB")
         mem_view = memoryview(ram_image)
-        image = np.frombuffer(mem_view, np.uint8)
-        image.shape = (camera.tex.getYSize(), camera.tex.getXSize(), 3)
+        image: np.ndarray = np.frombuffer(mem_view, np.uint8)
+        width, height = camera.image_dimensions
+        image.shape = (height, width, 3)
         image = np.flipud(image)
 
         metadata = GridMapMetadata(
             resolution=self._resolution,
             height=image.shape[0],
             width=image.shape[1],
-            camera_position=camera.camera_np.getPos(),
-            camera_heading_in_degrees=camera.camera_np.getH(),
+            camera_position=camera.position,
+            camera_heading=camera.heading,
         )
         return TopDownRGB(data=image, metadata=metadata)
 

smarts/core/sensor_manager.py

@@ -24,6 +24,7 @@
 from typing import Dict, FrozenSet, List, Optional, Set, Tuple
 
 from smarts.core import config
+from smarts.core.renderer_base import RendererBase
 from smarts.core.sensors import Observation, Sensor, Sensors, SensorState
 from smarts.core.sensors.local_sensor_resolver import LocalSensorResolver
 from smarts.core.sensors.parallel_sensor_resolver import ParallelSensorResolver
@@ -85,7 +86,7 @@ def observe(
         sim_frame,
         sim_local_constants,
         agent_ids,
-        renderer_ref,
+        renderer_ref: RendererBase,
         physics_ref,
     ):
         """Runs observations and updates the sensor states.
@@ -96,7 +97,7 @@ def observe(
                 The values that should stay the same for a simulation over a reset.
             agent_ids ({str, ...}):
                 The agent ids to process.
-            renderer_ref (Optional[Renderer]):
+            renderer_ref (Optional[RendererBase]):
                 The renderer (if any) that should be used.
             physics_ref (bc.BulletClient):
                 The physics client.