[backport foxy-future] Reindexer (#641, #699)

.gitignore

@@ -6,3 +6,5 @@ build/
 venv/
 **/.pytest_cache/
 __pycache__/
+*.db3-*
+

ros2bag/ros2bag/verb/reindex.py

@@ -0,0 +1,53 @@
+# Copyright 2021 DCS Corporation, All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# DISTRIBUTION A. Approved for public release; distribution unlimited.
+# OPSEC #4584.
+#
+# Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS
+# Part 252.227-7013 or 7014 (Feb 2014).
+#
+# This notice must appear in all copies of this file and its derivatives.
+
+import os
+
+from ros2bag.api import check_path_exists
+from ros2bag.api import print_error
+from ros2bag.verb import VerbExtension
+from rosbag2_py import get_registered_readers, Reindexer, StorageOptions
+
+
+class ReindexVerb(VerbExtension):
+    """Reconstruct metadata file for a bag."""
+
+    def add_arguments(self, parser, cli_name):
+        storage_choices = get_registered_readers()
+        default_storage = 'sqlite3' if 'sqlite3' in storage_choices else storage_choices[0]
+        parser.add_argument(
+            'bag_directory', type=check_path_exists, help='bag to reindex')
+        parser.add_argument(
+            'storage_id', default=default_storage, choices=storage_choices,
+            help=f"storage identifier to be used, defaults to '{default_storage}'")
+
+    def main(self, *, args):
+        if not os.path.isdir(args.bag_directory):
+            return print_error('Must specify a bag directory')
+
+        storage_options = StorageOptions(
+            uri=args.bag_directory,
+            storage_id=args.storage_id,
+        )
+
+        reindexer = Reindexer()
+        reindexer.reindex(storage_options)

ros2bag/setup.py

@@ -42,6 +42,7 @@
             'list = ros2bag.verb.list:ListVerb',
             'play = ros2bag.verb.play:PlayVerb',
             'record = ros2bag.verb.record:RecordVerb',
+            'reindex = ros2bag.verb.reindex:ReindexVerb'
         ],
     }
 )

rosbag2_compression/test/rosbag2_compression/mock_storage.hpp

@@ -48,6 +48,7 @@ class MockStorage : public rosbag2_storage::storage_interfaces::ReadWriteInterfa
   MOCK_CONST_METHOD0(get_bagfile_size, uint64_t());
   MOCK_CONST_METHOD0(get_relative_file_path, std::string());
   MOCK_CONST_METHOD0(get_storage_identifier, std::string());
+  MOCK_CONST_METHOD0(get_storage_extension, std::string());
   MOCK_CONST_METHOD0(get_minimum_split_file_size, uint64_t());
 };
 

rosbag2_cpp/CMakeLists.txt

@@ -1,6 +1,8 @@
 cmake_minimum_required(VERSION 3.5)
 project(rosbag2_cpp)
 
+add_definitions(-D_SRC_REINDEX_DIR_PATH="${CMAKE_CURRENT_SOURCE_DIR}/test/rosbag2_cpp/reindex_test_bags")
+
 # Default to C99
 if(NOT CMAKE_C_STANDARD)
   set(CMAKE_C_STANDARD 99)
@@ -61,9 +63,11 @@ add_library(${PROJECT_NAME} SHARED
   src/rosbag2_cpp/typesupport_helpers.cpp
   src/rosbag2_cpp/types/introspection_message.cpp
   src/rosbag2_cpp/writer.cpp
-  src/rosbag2_cpp/writers/sequential_writer.cpp)
+  src/rosbag2_cpp/writers/sequential_writer.cpp
+  src/rosbag2_cpp/reindexer.cpp)
 
 ament_target_dependencies(${PROJECT_NAME}
+  PUBLIC
   ament_index_cpp
   pluginlib
   rclcpp
@@ -188,6 +192,7 @@ if(BUILD_TESTING)
     ament_target_dependencies(test_message_cache rosbag2_storage)
   endif()
 
+
   # If compiling with gcc, run this test with sanitizers enabled
   ament_add_gmock(test_ros2_message
     test/rosbag2_cpp/types/test_ros2_message.cpp

rosbag2_cpp/include/rosbag2_cpp/reindexer.hpp

@@ -0,0 +1,128 @@
+// Copyright 2020 DCS Corporation, All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// DISTRIBUTION A. Approved for public release; distribution unlimited.
+// OPSEC #4584.
+//
+// Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS
+// Part 252.227-7013 or 7014 (Feb 2014).
+//
+// This notice must appear in all copies of this file and its derivatives.
+
+#ifndef ROSBAG2_CPP__REINDEXER_HPP_
+#define ROSBAG2_CPP__REINDEXER_HPP_
+
+#include <memory>
+#include <regex>
+#include <string>
+#include <vector>
+
+#include "rcpputils/filesystem_helper.hpp"
+
+#include "rosbag2_cpp/converter.hpp"
+#include "rosbag2_cpp/reader.hpp"
+#include "rosbag2_cpp/readers/sequential_reader.hpp"
+#include "rosbag2_cpp/serialization_format_converter_factory.hpp"
+#include "rosbag2_cpp/serialization_format_converter_factory_interface.hpp"
+#include "rosbag2_cpp/visibility_control.hpp"
+
+#include "rosbag2_storage/metadata_io.hpp"
+#include "rosbag2_storage/storage_factory.hpp"
+#include "rosbag2_storage/storage_factory_interface.hpp"
+#include "rosbag2_storage/storage_options.hpp"
+#include "rosbag2_storage/storage_filter.hpp"
+#include "rosbag2_storage/storage_interfaces/read_only_interface.hpp"
+
+// This is necessary because of using stl types here. It is completely safe, because
+// a) the member is not accessible from the outside
+// b) there are no inline functions.
+#ifdef _WIN32
+# pragma warning(push)
+# pragma warning(disable:4251)
+#endif
+
+namespace rosbag2_cpp
+{
+
+/**
+ * Tool to reconstruct bag metadata files in the event of loss or corruption
+ *
+ * Reindexing is an operation where a bag that is missing a metadata.yaml file can have a new
+ *   file created through parsing of the metadata stored within the actual files of the bag.
+ *   For instance: Imagine we are working with SQL databases (.db3). We can open the individual
+ *   .db3 files within the bag and read their metadata (not the messages themselves) to replicate
+ *   a usable metadata.yaml file, so that the bag can once again be read by the standard read
+ *   command.
+ *
+ * Reindexing has some limitations - It cannot perfectly replicate the original metadata file,
+ *   since some information known by the program from the start up command cannot be found
+ *   within the metadata. But it should at least repair a bag to the point it can be read
+ *   again.
+ *
+ */
+class ROSBAG2_CPP_PUBLIC Reindexer
+{
+public:
+  Reindexer(
+    std::unique_ptr<rosbag2_storage::StorageFactoryInterface> storage_factory =
+    std::make_unique<rosbag2_storage::StorageFactory>(),
+    std::unique_ptr<rosbag2_storage::MetadataIo> metadata_io =
+    std::make_unique<rosbag2_storage::MetadataIo>());
+
+  virtual ~Reindexer() = default;
+
+  /// Use the supplied storage options to reindex a bag defined by the storage options URI.
+  /*
+  * \param storage_options Provides best-guess parameters for the bag's original settings.
+  */
+  void reindex(const rosbag2_storage::StorageOptions & storage_options);
+
+protected:
+  std::unique_ptr<rosbag2_storage::StorageFactoryInterface> storage_factory_{};
+  std::unique_ptr<rosbag2_storage::MetadataIo> metadata_io_{};
+  rosbag2_storage::BagMetadata metadata_{};
+  std::vector<rosbag2_storage::TopicMetadata> topics_metadata_{};
+
+private:
+  std::string regex_bag_pattern_;
+  rcpputils::fs::path base_folder_;   // The folder that the bag files are in
+  std::shared_ptr<SerializationFormatConverterFactoryInterface> converter_factory_{};
+  void get_bag_files(
+    const rcpputils::fs::path & base_folder,
+    std::vector<rcpputils::fs::path> & output);
+
+  // Prepares the metadata by setting initial values.
+  void init_metadata(
+    const std::vector<rcpputils::fs::path> & files,
+    const rosbag2_storage::StorageOptions & storage_options);
+
+  // Attempts to harvest metadata from all bag files, and aggregates the result
+  void aggregate_metadata(
+    const std::vector<rcpputils::fs::path> & files,
+    const std::unique_ptr<rosbag2_cpp::readers::SequentialReader> & bag_reader,
+    const rosbag2_storage::StorageOptions & storage_options);
+
+  // Comparison function for std::sort with our filepath convention
+  bool compare_relative_file(
+    const rcpputils::fs::path & first_path,
+    const rcpputils::fs::path & second_path);
+};
+
+}  // namespace rosbag2_cpp
+
+#ifdef _WIN32
+# pragma warning(pop)
+#endif
+
+#endif  // ROSBAG2_CPP__REINDEXER_HPP_