added locking to prevent race condition in action client methods (send goal, ...) when the response arrives before the sequence number is written to the internal structures; see ros2#878

Author: jmarsik

rclpy/rclpy/action/client.py

@@ -139,6 +139,14 @@ def __init__(
         :param feedback_sub_qos_profile: QoS profile for the feedback subscriber.
         :param status_sub_qos_profile: QoS profile for the status subscriber.
         """
+        
+        # lock to avoid race conditions on shared state (_sequence_number_to_goal_id and others)
+        # without it the goal accepted response can arrive before its sequence number is placed
+        #  into the dictionary, ActionClient will drop the response in that case and the goal
+        #  will never complete!
+        # see https://github.com/ros2/rclpy/issues/878
+        self._internal_lock = threading.RLock()
+
         if callback_group is None:
             callback_group = node.default_callback_group
 
@@ -195,21 +203,23 @@ def _remove_pending_request(self, future, pending_requests):
         :return: The sequence number associated with the removed future, or
             None if the future was not found in the list.
         """
-        for seq, req_future in list(pending_requests.items()):
-            if future == req_future:
-                try:
-                    del pending_requests[seq]
-                except KeyError:
-                    pass
-                else:
-                    self.remove_future(future)
-                    return seq
+        with self._internal_lock:
+            for seq, req_future in list(pending_requests.items()):
+                if future == req_future:
+                    try:
+                        del pending_requests[seq]
+                    except KeyError:
+                        pass
+                    else:
+                        self.remove_future(future)
+                        return seq
         return None
 
     def _remove_pending_goal_request(self, future):
-        seq = self._remove_pending_request(future, self._pending_goal_requests)
-        if seq in self._sequence_number_to_goal_id:
-            del self._sequence_number_to_goal_id[seq]
+        with self._internal_lock:
+            seq = self._remove_pending_request(future, self._pending_goal_requests)
+            if seq in self._sequence_number_to_goal_id:
+                del self._sequence_number_to_goal_id[seq]
 
     def _remove_pending_cancel_request(self, future):
         self._remove_pending_request(future, self._pending_cancel_requests)
@@ -277,71 +287,76 @@ async def execute(self, taken_data):
         """
         if 'goal' in taken_data:
             sequence_number, goal_response = taken_data['goal']
-            if sequence_number in self._sequence_number_to_goal_id:
-                goal_handle = ClientGoalHandle(
-                    self,
-                    self._sequence_number_to_goal_id[sequence_number],
-                    goal_response)
-
-                if goal_handle.accepted:
-                    goal_uuid = bytes(goal_handle.goal_id.uuid)
-                    if goal_uuid in self._goal_handles:
-                        raise RuntimeError(
-                            'Two goals were accepted with the same ID ({})'.format(goal_handle))
-                    self._goal_handles[goal_uuid] = weakref.ref(goal_handle)
-
-                self._pending_goal_requests[sequence_number].set_result(goal_handle)
-            else:
-                self._node.get_logger().warning(
-                    'Ignoring unexpected goal response. There may be more than '
-                    f"one action server for the action '{self._action_name}'"
-                )
+            with self._internal_lock:
+                if sequence_number in self._sequence_number_to_goal_id:
+                    goal_handle = ClientGoalHandle(
+                        self,
+                        self._sequence_number_to_goal_id[sequence_number],
+                        goal_response)
+
+                    if goal_handle.accepted:
+                        goal_uuid = bytes(goal_handle.goal_id.uuid)
+                        if goal_uuid in self._goal_handles:
+                            raise RuntimeError(
+                                'Two goals were accepted with the same ID ({})'.format(goal_handle))
+                        self._goal_handles[goal_uuid] = weakref.ref(goal_handle)
+
+                    self._pending_goal_requests[sequence_number].set_result(goal_handle)
+                else:
+                    self._node.get_logger().warning(
+                        'Ignoring unexpected goal response. There may be more than '
+                        f"one action server for the action '{self._action_name}'"
+                    )
 
         if 'cancel' in taken_data:
             sequence_number, cancel_response = taken_data['cancel']
-            if sequence_number in self._pending_cancel_requests:
-                self._pending_cancel_requests[sequence_number].set_result(cancel_response)
-            else:
-                self._node.get_logger().warning(
-                    'Ignoring unexpected cancel response. There may be more than '
-                    f"one action server for the action '{self._action_name}'"
-                )
+            with self._internal_lock:
+                if sequence_number in self._pending_cancel_requests:
+                    self._pending_cancel_requests[sequence_number].set_result(cancel_response)
+                else:
+                    self._node.get_logger().warning(
+                        'Ignoring unexpected cancel response. There may be more than '
+                        f"one action server for the action '{self._action_name}'"
+                    )
 
         if 'result' in taken_data:
             sequence_number, result_response = taken_data['result']
-            if sequence_number in self._pending_result_requests:
-                self._pending_result_requests[sequence_number].set_result(result_response)
-            else:
-                self._node.get_logger().warning(
-                    'Ignoring unexpected result response. There may be more than '
-                    f"one action server for the action '{self._action_name}'"
-                )
+            with self._internal_lock:
+                if sequence_number in self._pending_result_requests:
+                    self._pending_result_requests[sequence_number].set_result(result_response)
+                else:
+                    self._node.get_logger().warning(
+                        'Ignoring unexpected result response. There may be more than '
+                        f"one action server for the action '{self._action_name}'"
+                    )
 
         if 'feedback' in taken_data:
             feedback_msg = taken_data['feedback']
             goal_uuid = bytes(feedback_msg.goal_id.uuid)
             # Call a registered callback if there is one
-            if goal_uuid in self._feedback_callbacks:
-                await await_or_execute(self._feedback_callbacks[goal_uuid], feedback_msg)
+            with self._internal_lock:
+                if goal_uuid in self._feedback_callbacks:
+                    await await_or_execute(self._feedback_callbacks[goal_uuid], feedback_msg)
 
         if 'status' in taken_data:
             # Update the status of all goal handles maintained by this Action Client
             for status_msg in taken_data['status'].status_list:
                 goal_uuid = bytes(status_msg.goal_info.goal_id.uuid)
                 status = status_msg.status
 
-                if goal_uuid in self._goal_handles:
-                    goal_handle = self._goal_handles[goal_uuid]()
-                    if goal_handle is not None:
-                        goal_handle._status = status
-                        # Remove "done" goals from the list
-                        if (GoalStatus.STATUS_SUCCEEDED == status or
-                                GoalStatus.STATUS_CANCELED == status or
-                                GoalStatus.STATUS_ABORTED == status):
+                with self._internal_lock:
+                    if goal_uuid in self._goal_handles:
+                        goal_handle = self._goal_handles[goal_uuid]()
+                        if goal_handle is not None:
+                            goal_handle._status = status
+                            # Remove "done" goals from the list
+                            if (GoalStatus.STATUS_SUCCEEDED == status or
+                                    GoalStatus.STATUS_CANCELED == status or
+                                    GoalStatus.STATUS_ABORTED == status):
+                                del self._goal_handles[goal_uuid]
+                        else:
+                            # Weak reference is None
                             del self._goal_handles[goal_uuid]
-                    else:
-                        # Weak reference is None
-                        del self._goal_handles[goal_uuid]
 
     def get_num_entities(self):
         """Return number of each type of entity used in the wait set."""
@@ -421,22 +436,23 @@ def send_goal_async(self, goal, feedback_callback=None, goal_uuid=None):
         request = self._action_type.Impl.SendGoalService.Request()
         request.goal_id = self._generate_random_uuid() if goal_uuid is None else goal_uuid
         request.goal = goal
-        sequence_number = self._client_handle.send_goal_request(request)
-        if sequence_number in self._pending_goal_requests:
-            raise RuntimeError(
-                'Sequence ({}) conflicts with pending goal request'.format(sequence_number))
-
-        if feedback_callback is not None:
-            # TODO(jacobperron): Move conversion function to a general-use package
-            goal_uuid = bytes(request.goal_id.uuid)
-            self._feedback_callbacks[goal_uuid] = feedback_callback
-
-        future = Future()
-        self._pending_goal_requests[sequence_number] = future
-        self._sequence_number_to_goal_id[sequence_number] = request.goal_id
-        future.add_done_callback(self._remove_pending_goal_request)
-        # Add future so executor is aware
-        self.add_future(future)
+        with self._internal_lock:
+            sequence_number = self._client_handle.send_goal_request(request)
+            if sequence_number in self._pending_goal_requests:
+                raise RuntimeError(
+                    'Sequence ({}) conflicts with pending goal request'.format(sequence_number))
+
+            if feedback_callback is not None:
+                # TODO(jacobperron): Move conversion function to a general-use package
+                goal_uuid = bytes(request.goal_id.uuid)
+                self._feedback_callbacks[goal_uuid] = feedback_callback
+
+            future = Future()
+            self._pending_goal_requests[sequence_number] = future
+            self._sequence_number_to_goal_id[sequence_number] = request.goal_id
+            future.add_done_callback(self._remove_pending_goal_request)
+            # Add future so executor is aware
+            self.add_future(future)
 
         return future
 
@@ -479,16 +495,17 @@ def _cancel_goal_async(self, goal_handle):
 
         cancel_request = CancelGoal.Request()
         cancel_request.goal_info.goal_id = goal_handle.goal_id
-        sequence_number = self._client_handle.send_cancel_request(cancel_request)
-        if sequence_number in self._pending_cancel_requests:
-            raise RuntimeError(
-                'Sequence ({}) conflicts with pending cancel request'.format(sequence_number))
+        with self._internal_lock:
+            sequence_number = self._client_handle.send_cancel_request(cancel_request)
+            if sequence_number in self._pending_cancel_requests:
+                raise RuntimeError(
+                    'Sequence ({}) conflicts with pending cancel request'.format(sequence_number))
 
-        future = Future()
-        self._pending_cancel_requests[sequence_number] = future
-        future.add_done_callback(self._remove_pending_cancel_request)
-        # Add future so executor is aware
-        self.add_future(future)
+            future = Future()
+            self._pending_cancel_requests[sequence_number] = future
+            future.add_done_callback(self._remove_pending_cancel_request)
+            # Add future so executor is aware
+            self.add_future(future)
 
         return future
 
@@ -531,16 +548,17 @@ def _get_result_async(self, goal_handle):
 
         result_request = self._action_type.Impl.GetResultService.Request()
         result_request.goal_id = goal_handle.goal_id
-        sequence_number = self._client_handle.send_result_request(result_request)
-        if sequence_number in self._pending_result_requests:
-            raise RuntimeError(
-                'Sequence ({}) conflicts with pending result request'.format(sequence_number))
-
-        future = Future()
-        self._pending_result_requests[sequence_number] = future
-        future.add_done_callback(self._remove_pending_result_request)
-        # Add future so executor is aware
-        self.add_future(future)
+        with self._internal_lock:
+            sequence_number = self._client_handle.send_result_request(result_request)
+            if sequence_number in self._pending_result_requests:
+                raise RuntimeError(
+                    'Sequence ({}) conflicts with pending result request'.format(sequence_number))
+
+            future = Future()
+            self._pending_result_requests[sequence_number] = future
+            future.add_done_callback(self._remove_pending_result_request)
+            # Add future so executor is aware
+            self.add_future(future)
 
         return future