Semantic Labeling

.gitignore

@@ -2,6 +2,9 @@
 build/
 __pycache__/
 
+# Environment Variables file
+.env
+
 # Compiled Object files
 *.slo
 *.lo

ada_feeding/ada_feeding/behaviors/acquisition/compute_food_frame.py

@@ -336,7 +336,7 @@ def update(self) -> py_trees.common.Status:
             x_unit.vector, x_pos.vector
         )
 
-        # # If you need to send a fixed food frame to the robot arm, e.g., to 
+        # # If you need to send a fixed food frame to the robot arm, e.g., to
         # # debug off-centering issues, uncomment this and modify the translation.
         # deg = 90  # fork roll
         # world_to_food_transform.transform.translation.x = 0.26262263022586224

ada_feeding_msgs/CMakeLists.txt

@@ -35,6 +35,7 @@ rosidl_generate_interfaces(${PROJECT_NAME}
   "srv/AcquisitionReport.srv"
   "srv/AcquisitionSelect.srv"
   "srv/GetRobotState.srv"
+  "srv/GenerateCaption.srv"
   "srv/ModifyCollisionObject.srv"
 
   DEPENDENCIES geometry_msgs sensor_msgs std_msgs

ada_feeding_msgs/action/SegmentAllItems.action

@@ -1,6 +1,8 @@
 # The interface for an action that gets an image from the camera and returns
 # the masks of all segmented items within that image.
 
+# The list of input semantic labels for the food items on the plate
+string caption   
 ---
 # Possible return statuses
 uint8 STATUS_SUCCEEDED=0
@@ -17,6 +19,9 @@ std_msgs/Header header
 sensor_msgs/CameraInfo camera_info
 # Masks of all the detected items in the image
 ada_feeding_msgs/Mask[] detected_items
+# A list of semantic labels corresponding to each of the masks of detected
+# items in the image
+string[] item_labels
 ---
 # How much time the action has spent segmenting the food item
 builtin_interfaces/Duration elapsed_time

ada_feeding_msgs/msg/Mask.msg

@@ -19,6 +19,9 @@ float64 average_depth
 # An arbitrary ID that defines the segmented item
 string item_id
 
+# An ID that semantically labels a specific, segmented item
+string object_id
+
 # A score that indicates how confident the segemntation algorithm is in 
 # this mask.
 float64 confidence

ada_feeding_msgs/srv/GenerateCaption.srv

@@ -0,0 +1,11 @@
+# The interface for a service that takes in a list of input labels
+# describing the food items on a plate and returns a sentence caption compiling
+# these labels used as a query for GroundingDINO detection.
+
+# A list of semantic labels corresponding to each of the masks of detected
+# items in the image
+string[] input_labels
+---
+# A sentence caption compiling the semantic labels used as a query for 
+# GroundingDINO to perform bounding box detections.
+string caption

ada_feeding_perception/ada_feeding_perception/ada_feeding_perception_node.py

@@ -170,6 +170,7 @@ def main(args=None):
     # pylint: disable=import-outside-toplevel
     from ada_feeding_perception.face_detection import FaceDetectionNode
     from ada_feeding_perception.food_on_fork_detection import FoodOnForkDetectionNode
+    from ada_feeding_perception.segment_all_items import SegmentAllItemsNode
     from ada_feeding_perception.segment_from_point import SegmentFromPointNode
     from ada_feeding_perception.table_detection import TableDetectionNode
 
@@ -178,6 +179,7 @@ def main(args=None):
     node = ADAFeedingPerceptionNode("ada_feeding_perception")
     face_detection = FaceDetectionNode(node)
     food_on_fork_detection = FoodOnForkDetectionNode(node)
+    segment_all_items = SegmentAllItemsNode(node)  # pylint: disable=unused-variable
     segment_from_point = SegmentFromPointNode(node)  # pylint: disable=unused-variable
     table_detection = TableDetectionNode(node)
     executor = MultiThreadedExecutor(num_threads=16)