Introduce ros2 bag reindex

.gitignore

@@ -5,3 +5,4 @@ cmake-build-release/
 build/
 venv/
 **/.pytest_cache/
+**/__pycache__/**

README.md

@@ -174,6 +174,14 @@ Topic information: Topic: /chatter | Type: std_msgs/String | Count: 9 | Serializ
                    Topic: /my_chatter | Type: std_msgs/String | Count: 18 | Serialization Format: cdr
 ```
 
+### Reindexing Data
+
+In the event that a bag file becomes unreadable, due to either a corrupted or missing `metadata.yaml` file within the bag, you can attempt to recover the bag by reindexing:
+```
+$ ros2 bag reindex <bag_file>
+```
+This will attempt to read the internal storage of the bag file to reconstruct a workable `metadata.yaml` file. It is not guaranteed to be the exact same as a file generated during the recording process. You may have to check the file manually to make sure all the details are correct.
+
 ### Overriding QoS Profiles
 
 When starting a recording or playback workflow, you can pass a YAML file that contains QoS profile settings for a specific topic.

ros2bag/ros2bag/verb/reindex.py

@@ -0,0 +1,58 @@
+# Copyright 2018 Open Source Robotics Foundation, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from ros2bag.api import check_path_exists
+from ros2bag.verb import VerbExtension
+
+
+class ReindexVerb(VerbExtension):
+    """Generate metadata from a bag."""
+
+    def add_arguments(self, parser, cli_name):  # noqa: D102
+        parser.add_argument(
+            'bag_file', type=check_path_exists, help='Bag file to reindex')
+        parser.add_argument(
+            '-s', '--storage', default='sqlite3',
+            help="storage identifier to be used, defaults to 'sqlite3'")
+        parser.add_argument(
+            '-f', '--serialization-format', default='',
+            help='rmw serialization format in which the messages are saved, defaults to the'
+                 ' rmw currently in use')
+        parser.add_argument(
+            '--compression-format', type=str, default='', choices=['zstd'],
+            help='Specify the compression format/algorithm. Default is none.'
+        )
+        self._subparser = parser
+
+    def main(self, *, args):  # noqa: D102
+        uri = args.bag_file
+
+        # NOTE(hidmic): in merged install workspaces on Windows, Python entrypoint lookups
+        #               combined with constrained environments (as imposed by colcon test)
+        #               may result in DLL loading failures when attempting to import a C
+        #               extension. Therefore, do not import rosbag2_transport at the module
+        #               level but on demand, right before first use.
+        from rosbag2_transport import rosbag2_transport_py
+
+        rosbag2_transport_py.reindex(
+            uri=uri,
+            storage_id=args.storage,
+            serialization_format=args.serialization_format,
+            compression_format=args.compression_format
+        )
+
+        if os.path.isdir(uri) and not os.listdir(uri):
+            os.rmdir(uri)

ros2bag/setup.py

@@ -42,6 +42,7 @@
             'list = ros2bag.verb.list:ListVerb',
             'play = ros2bag.verb.play:PlayVerb',
             'record = ros2bag.verb.record:RecordVerb',
+            'reindex = ros2bag.verb.reindex:ReindexVerb'
         ],
     }
 )

ros2bag/test/resources/reindex_test_bags/file_compression/file_compression_target.yaml

@@ -0,0 +1,34 @@
+rosbag2_bagfile_information:
+  version: 4
+  storage_identifier: sqlite3
+  relative_file_paths:
+    - file_compression/file_compression_0.db3.zstd
+    - file_compression/file_compression_1.db3.zstd
+    - file_compression/file_compression_2.db3.zstd
+    - file_compression/file_compression_3.db3.zstd
+  duration:
+    nanoseconds: 59997866716
+  starting_time:
+    nanoseconds_since_epoch: 1604357089919704830
+  message_count: 775
+  topics_with_message_count:
+    - topic_metadata:
+        name: /topic
+        type: std_msgs/msg/String
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 2147483647\n    nsec: 4294967295\n  lifespan:\n    sec: 2147483647\n    nsec: 4294967295\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 2147483647\n    nsec: 4294967295\n  avoid_ros_namespace_conventions: false"
+      message_count: 383
+    - topic_metadata:
+        name: /parameter_events
+        type: rcl_interfaces/msg/ParameterEvent
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 2147483647\n    nsec: 4294967295\n  lifespan:\n    sec: 2147483647\n    nsec: 4294967295\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 2147483647\n    nsec: 4294967295\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /rosout
+        type: rcl_interfaces/msg/Log
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 1\n  deadline:\n    sec: 2147483647\n    nsec: 4294967295\n  lifespan:\n    sec: 10\n    nsec: 0\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 2147483647\n    nsec: 4294967295\n  avoid_ros_namespace_conventions: false"
+      message_count: 392
+  compression_format: zstd
+  compression_mode: FILE

ros2bag/test/resources/reindex_test_bags/message_compression/message_compression_target.yaml

@@ -0,0 +1,34 @@
+rosbag2_bagfile_information:
+  version: 4
+  storage_identifier: sqlite3
+  relative_file_paths:
+    - message_compression/message_compression_0.db3
+    - message_compression/message_compression_1.db3
+    - message_compression/message_compression_2.db3
+    - message_compression/message_compression_3.db3
+  duration:
+    nanoseconds: 59872587605
+  starting_time:
+    nanoseconds_since_epoch: 1604581538133709023
+  message_count: 1127
+  topics_with_message_count:
+    - topic_metadata:
+        name: /parameter_events
+        type: rcl_interfaces/msg/ParameterEvent
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 2147483647\n    nsec: 4294967295\n  lifespan:\n    sec: 2147483647\n    nsec: 4294967295\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 2147483647\n    nsec: 4294967295\n  avoid_ros_namespace_conventions: false\n- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 2147483647\n    nsec: 4294967295\n  lifespan:\n    sec: 2147483647\n    nsec: 4294967295\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 2147483647\n    nsec: 4294967295\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /rosout
+        type: rcl_interfaces/msg/Log
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 1\n  deadline:\n    sec: 2147483647\n    nsec: 4294967295\n  lifespan:\n    sec: 10\n    nsec: 0\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 2147483647\n    nsec: 4294967295\n  avoid_ros_namespace_conventions: false\n- history: 3\n  depth: 0\n  reliability: 1\n  durability: 1\n  deadline:\n    sec: 2147483647\n    nsec: 4294967295\n  lifespan:\n    sec: 10\n    nsec: 0\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 2147483647\n    nsec: 4294967295\n  avoid_ros_namespace_conventions: false"
+      message_count: 758
+    - topic_metadata:
+        name: /topic
+        type: std_msgs/msg/String
+        serialization_format: cdr
+        offered_qos_profiles: ""
+      message_count: 369
+  compression_format: zstd
+  compression_mode: MESSAGE

ros2bag/test/resources/reindex_test_bags/multiple_files/multiple_files_target.yaml

@@ -0,0 +1,34 @@
+rosbag2_bagfile_information:
+  version: 4
+  storage_identifier: sqlite3
+  relative_file_paths:
+    - multiple_files_0.db3
+    - multiple_files_1.db3
+    - multiple_files_2.db3
+    - multiple_files_3.db3
+  duration:
+    nanoseconds: 59997863418
+  starting_time:
+    nanoseconds_since_epoch: 1604356847420232242
+  message_count: 766
+  topics_with_message_count:
+    - topic_metadata:
+        name: /topic
+        type: std_msgs/msg/String
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 2147483647\n    nsec: 4294967295\n  lifespan:\n    sec: 2147483647\n    nsec: 4294967295\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 2147483647\n    nsec: 4294967295\n  avoid_ros_namespace_conventions: false"
+      message_count: 378
+    - topic_metadata:
+        name: /parameter_events
+        type: rcl_interfaces/msg/ParameterEvent
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 2147483647\n    nsec: 4294967295\n  lifespan:\n    sec: 2147483647\n    nsec: 4294967295\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 2147483647\n    nsec: 4294967295\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /rosout
+        type: rcl_interfaces/msg/Log
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 1\n  deadline:\n    sec: 2147483647\n    nsec: 4294967295\n  lifespan:\n    sec: 10\n    nsec: 0\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 2147483647\n    nsec: 4294967295\n  avoid_ros_namespace_conventions: false"
+      message_count: 388
+  compression_format: ""
+  compression_mode: ""