diff --git a/.gitignore b/.gitignore index d35fe720ad..714bd826b0 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ build/ venv/ **/.pytest_cache/ __pycache__/ +*.db3-* + diff --git a/ros2bag/ros2bag/verb/reindex.py b/ros2bag/ros2bag/verb/reindex.py new file mode 100644 index 0000000000..83ccca4c59 --- /dev/null +++ b/ros2bag/ros2bag/verb/reindex.py @@ -0,0 +1,53 @@ +# Copyright 2021 DCS Corporation, All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DISTRIBUTION A. Approved for public release; distribution unlimited. +# OPSEC #4584. +# +# Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS +# Part 252.227-7013 or 7014 (Feb 2014). +# +# This notice must appear in all copies of this file and its derivatives. + +import os + +from ros2bag.api import check_path_exists +from ros2bag.api import print_error +from ros2bag.verb import VerbExtension +from rosbag2_py import get_registered_readers, Reindexer, StorageOptions + + +class ReindexVerb(VerbExtension): + """Reconstruct metadata file for a bag.""" + + def add_arguments(self, parser, cli_name): + storage_choices = get_registered_readers() + default_storage = 'sqlite3' if 'sqlite3' in storage_choices else storage_choices[0] + parser.add_argument( + 'bag_directory', type=check_path_exists, help='bag to reindex') + parser.add_argument( + 'storage_id', default=default_storage, choices=storage_choices, + help=f"storage identifier to be used, defaults to '{default_storage}'") + + def main(self, *, args): + if not os.path.isdir(args.bag_directory): + return print_error('Must specify a bag directory') + + storage_options = StorageOptions( + uri=args.bag_directory, + storage_id=args.storage_id, + ) + + reindexer = Reindexer() + reindexer.reindex(storage_options) diff --git a/ros2bag/setup.py b/ros2bag/setup.py index c409f8bee3..40cbd99e4c 100644 --- a/ros2bag/setup.py +++ b/ros2bag/setup.py @@ -42,6 +42,7 @@ 'list = ros2bag.verb.list:ListVerb', 'play = ros2bag.verb.play:PlayVerb', 'record = ros2bag.verb.record:RecordVerb', + 'reindex = ros2bag.verb.reindex:ReindexVerb' ], } ) diff --git a/rosbag2_compression/test/rosbag2_compression/mock_storage.hpp b/rosbag2_compression/test/rosbag2_compression/mock_storage.hpp index 3feb383d07..39df735d41 100644 --- a/rosbag2_compression/test/rosbag2_compression/mock_storage.hpp +++ b/rosbag2_compression/test/rosbag2_compression/mock_storage.hpp @@ -48,6 +48,7 @@ class MockStorage : public rosbag2_storage::storage_interfaces::ReadWriteInterfa MOCK_CONST_METHOD0(get_bagfile_size, uint64_t()); MOCK_CONST_METHOD0(get_relative_file_path, std::string()); MOCK_CONST_METHOD0(get_storage_identifier, std::string()); + MOCK_CONST_METHOD0(get_storage_extension, std::string()); MOCK_CONST_METHOD0(get_minimum_split_file_size, uint64_t()); }; diff --git a/rosbag2_cpp/CMakeLists.txt b/rosbag2_cpp/CMakeLists.txt index f93b486ac0..8e3d513923 100644 --- a/rosbag2_cpp/CMakeLists.txt +++ b/rosbag2_cpp/CMakeLists.txt @@ -1,6 +1,8 @@ cmake_minimum_required(VERSION 3.5) project(rosbag2_cpp) +add_definitions(-D_SRC_REINDEX_DIR_PATH="${CMAKE_CURRENT_SOURCE_DIR}/test/rosbag2_cpp/reindex_test_bags") + # Default to C99 if(NOT CMAKE_C_STANDARD) set(CMAKE_C_STANDARD 99) @@ -61,9 +63,11 @@ add_library(${PROJECT_NAME} SHARED src/rosbag2_cpp/typesupport_helpers.cpp src/rosbag2_cpp/types/introspection_message.cpp src/rosbag2_cpp/writer.cpp - src/rosbag2_cpp/writers/sequential_writer.cpp) + src/rosbag2_cpp/writers/sequential_writer.cpp + src/rosbag2_cpp/reindexer.cpp) ament_target_dependencies(${PROJECT_NAME} + PUBLIC ament_index_cpp pluginlib rclcpp @@ -188,6 +192,7 @@ if(BUILD_TESTING) ament_target_dependencies(test_message_cache rosbag2_storage) endif() + # If compiling with gcc, run this test with sanitizers enabled ament_add_gmock(test_ros2_message test/rosbag2_cpp/types/test_ros2_message.cpp diff --git a/rosbag2_cpp/include/rosbag2_cpp/reindexer.hpp b/rosbag2_cpp/include/rosbag2_cpp/reindexer.hpp new file mode 100644 index 0000000000..6d5b54fd51 --- /dev/null +++ b/rosbag2_cpp/include/rosbag2_cpp/reindexer.hpp @@ -0,0 +1,128 @@ +// Copyright 2020 DCS Corporation, All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// DISTRIBUTION A. Approved for public release; distribution unlimited. +// OPSEC #4584. +// +// Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS +// Part 252.227-7013 or 7014 (Feb 2014). +// +// This notice must appear in all copies of this file and its derivatives. + +#ifndef ROSBAG2_CPP__REINDEXER_HPP_ +#define ROSBAG2_CPP__REINDEXER_HPP_ + +#include +#include +#include +#include + +#include "rcpputils/filesystem_helper.hpp" + +#include "rosbag2_cpp/converter.hpp" +#include "rosbag2_cpp/reader.hpp" +#include "rosbag2_cpp/readers/sequential_reader.hpp" +#include "rosbag2_cpp/serialization_format_converter_factory.hpp" +#include "rosbag2_cpp/serialization_format_converter_factory_interface.hpp" +#include "rosbag2_cpp/visibility_control.hpp" + +#include "rosbag2_storage/metadata_io.hpp" +#include "rosbag2_storage/storage_factory.hpp" +#include "rosbag2_storage/storage_factory_interface.hpp" +#include "rosbag2_storage/storage_options.hpp" +#include "rosbag2_storage/storage_filter.hpp" +#include "rosbag2_storage/storage_interfaces/read_only_interface.hpp" + +// This is necessary because of using stl types here. It is completely safe, because +// a) the member is not accessible from the outside +// b) there are no inline functions. +#ifdef _WIN32 +# pragma warning(push) +# pragma warning(disable:4251) +#endif + +namespace rosbag2_cpp +{ + +/** + * Tool to reconstruct bag metadata files in the event of loss or corruption + * + * Reindexing is an operation where a bag that is missing a metadata.yaml file can have a new + * file created through parsing of the metadata stored within the actual files of the bag. + * For instance: Imagine we are working with SQL databases (.db3). We can open the individual + * .db3 files within the bag and read their metadata (not the messages themselves) to replicate + * a usable metadata.yaml file, so that the bag can once again be read by the standard read + * command. + * + * Reindexing has some limitations - It cannot perfectly replicate the original metadata file, + * since some information known by the program from the start up command cannot be found + * within the metadata. But it should at least repair a bag to the point it can be read + * again. + * + */ +class ROSBAG2_CPP_PUBLIC Reindexer +{ +public: + Reindexer( + std::unique_ptr storage_factory = + std::make_unique(), + std::unique_ptr metadata_io = + std::make_unique()); + + virtual ~Reindexer() = default; + + /// Use the supplied storage options to reindex a bag defined by the storage options URI. + /* + * \param storage_options Provides best-guess parameters for the bag's original settings. + */ + void reindex(const rosbag2_storage::StorageOptions & storage_options); + +protected: + std::unique_ptr storage_factory_{}; + std::unique_ptr metadata_io_{}; + rosbag2_storage::BagMetadata metadata_{}; + std::vector topics_metadata_{}; + +private: + std::string regex_bag_pattern_; + rcpputils::fs::path base_folder_; // The folder that the bag files are in + std::shared_ptr converter_factory_{}; + void get_bag_files( + const rcpputils::fs::path & base_folder, + std::vector & output); + + // Prepares the metadata by setting initial values. + void init_metadata( + const std::vector & files, + const rosbag2_storage::StorageOptions & storage_options); + + // Attempts to harvest metadata from all bag files, and aggregates the result + void aggregate_metadata( + const std::vector & files, + const std::unique_ptr & bag_reader, + const rosbag2_storage::StorageOptions & storage_options); + + // Comparison function for std::sort with our filepath convention + bool compare_relative_file( + const rcpputils::fs::path & first_path, + const rcpputils::fs::path & second_path); +}; + +} // namespace rosbag2_cpp + +#ifdef _WIN32 +# pragma warning(pop) +#endif + +#endif // ROSBAG2_CPP__REINDEXER_HPP_ diff --git a/rosbag2_cpp/src/rosbag2_cpp/reindexer.cpp b/rosbag2_cpp/src/rosbag2_cpp/reindexer.cpp new file mode 100644 index 0000000000..06df36ef89 --- /dev/null +++ b/rosbag2_cpp/src/rosbag2_cpp/reindexer.cpp @@ -0,0 +1,448 @@ +// Copyright 2020 DCS Corporation, All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// DISTRIBUTION A. Approved for public release; distribution unlimited. +// OPSEC #4584. +// +// Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS +// Part 252.227-7013 or 7014 (Feb 2014). +// +// This notice must appear in all copies of this file and its derivatives. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rcpputils/asserts.hpp" +#include "rcpputils/filesystem_helper.hpp" + +#include "rcutils/filesystem.h" + +#include "rosbag2_cpp/logging.hpp" +#include "rosbag2_cpp/reader.hpp" +#include "rosbag2_cpp/reindexer.hpp" + +#include "rosbag2_storage/storage_options.hpp" + +/// Foxy compatibility. Adapt for lack of https://github.com/ros2/rcutils/pull/323 +namespace +{ +extern "C" +{ +/* *INDENT-OFF* */ +/// An iterator used for enumerating directory contents +typedef struct rcutils_dir_iter_t +{ + /// The name of the enumerated file or directory + const char * entry_name; + /// The allocator used internally by iteration functions + rcutils_allocator_t allocator; + /// The platform-specific iteration state + void * state; +} rcutils_dir_iter_t; + +typedef struct rcutils_dir_iter_state_t +{ +#ifdef _WIN32 + HANDLE handle; + WIN32_FIND_DATA data; +#else + DIR * dir; +#endif +} rcutils_dir_iter_state_t; + +/// Begin iterating over the contents of the specified directory. +/** + * This function is used to list the files and directories that are contained in + * a specified directory. The structure returned by it must be deallocated using + * ::rcutils_dir_iter_end when the iteration is completed. The name of the + * enumerated entry is stored in the `entry_name` member of the returned object, + * and the first entry is already populated upon completion of this function. To + * populate the entry with the name of the next entry, use the + * ::rcutils_dir_iter_next function. Note that the "." and ".." entries are + * typically among the entries enumerated. + * \param[in] directory_path The directory path to iterate over the contents of. + * \param[in] allocator Allocator used to create the returned structure. + * \return An iterator object used to continue iterating directory contents + * \return NULL if an error occurred + */ +rcutils_dir_iter_t * +rcutils_dir_iter_start(const char * directory_path, const rcutils_allocator_t allocator); + +/// Continue iterating over the contents of a directory. +/** + * \param[in] iter An iterator created by ::rcutils_dir_iter_start. + * \return `true` if another entry was found, or + * \return `false` if there are no more entries in the directory. + */ +bool +rcutils_dir_iter_next(rcutils_dir_iter_t * iter); + +/// Finish iterating over the contents of a directory. +/** + * \param[in] iter An iterator created by ::rcutils_dir_iter_start. + */ +void +rcutils_dir_iter_end(rcutils_dir_iter_t * iter); + +rcutils_dir_iter_t * +rcutils_dir_iter_start(const char * directory_path, const rcutils_allocator_t allocator) +{ + RCUTILS_CHECK_ARGUMENT_FOR_NULL(directory_path, NULL); + RCUTILS_CHECK_ALLOCATOR_WITH_MSG( + &allocator, "allocator is invalid", return NULL); + struct dirent * entry = NULL; + + rcutils_dir_iter_t * iter = (rcutils_dir_iter_t *)allocator.zero_allocate( // NOLINT + 1, sizeof(rcutils_dir_iter_t), allocator.state); + if (NULL == iter) { + return NULL; + } + iter->allocator = allocator; + + rcutils_dir_iter_state_t * state = (rcutils_dir_iter_state_t *)allocator.zero_allocate( // NOLINT + 1, sizeof(rcutils_dir_iter_state_t), allocator.state); + if (NULL == state) { + RCUTILS_SET_ERROR_MSG( + "Failed to allocate memory.\n"); + goto rcutils_dir_iter_start_fail; + } + iter->state = (void *)state; // NOLINT + +#ifdef _WIN32 + char * search_path = rcutils_join_path(directory_path, "*", allocator); + if (NULL == search_path) { + goto rcutils_dir_iter_start_fail; + } + state->handle = FindFirstFile(search_path, &state->data); + allocator.deallocate(search_path, allocator.state); + if (INVALID_HANDLE_VALUE == state->handle) { + DWORD error = GetLastError(); + if (ERROR_FILE_NOT_FOUND != error || !rcutils_is_directory(directory_path)) { + RCUTILS_SET_ERROR_MSG_WITH_FORMAT_STRING( + "Can't open directory %s. Error code: %d\n", directory_path, error); + goto rcutils_dir_iter_start_fail; + } + } else { + iter->entry_name = state->data.cFileName; + } +#else + state->dir = opendir(directory_path); + if (NULL == state->dir) { + RCUTILS_SET_ERROR_MSG_WITH_FORMAT_STRING( + "Can't open directory %s. Error code: %d\n", directory_path, errno); + goto rcutils_dir_iter_start_fail; + } + + errno = 0; + entry = readdir(state->dir); + if (NULL != entry) { + iter->entry_name = entry->d_name; + } else if (0 != errno) { + RCUTILS_SET_ERROR_MSG_WITH_FORMAT_STRING( + "Can't iterate directory %s. Error code: %d\n", directory_path, errno); + goto rcutils_dir_iter_start_fail; + } +#endif + + return iter; + +rcutils_dir_iter_start_fail: + rcutils_dir_iter_end(iter); + return NULL; +} + +bool +rcutils_dir_iter_next(rcutils_dir_iter_t * iter) +{ + RCUTILS_CHECK_ARGUMENT_FOR_NULL(iter, false); + rcutils_dir_iter_state_t * state = (rcutils_dir_iter_state_t *)iter->state; // NOLINT + RCUTILS_CHECK_FOR_NULL_WITH_MSG(state, "iter is invalid", false); + +#ifdef _WIN32 + if (FindNextFile(state->handle, &state->data)) { + iter->entry_name = state->data.cFileName; + return true; + } + FindClose(state->handle); +#else + struct dirent * entry = readdir(state->dir); + if (NULL != entry) { + iter->entry_name = entry->d_name; + return true; + } +#endif + + iter->entry_name = NULL; + return false; +} + +void +rcutils_dir_iter_end(rcutils_dir_iter_t * iter) +{ + if (NULL == iter) { + return; + } + + rcutils_allocator_t allocator = iter->allocator; + rcutils_dir_iter_state_t * state = (rcutils_dir_iter_state_t *)iter->state; // NOLINT + if (NULL != state) { +#ifdef _WIN32 + FindClose(state->handle); +#else + if (NULL != state->dir) { + closedir(state->dir); + } +#endif + + allocator.deallocate(state, allocator.state); + } + + allocator.deallocate(iter, allocator.state); +} +/* *INDENT-ON* */ +} // extern "C" +} // namespace +/// End Foxy compatibility section. + +namespace rosbag2_cpp +{ +Reindexer::Reindexer( + std::unique_ptr storage_factory, + std::unique_ptr metadata_io) +: storage_factory_(std::move(storage_factory)), + metadata_io_(std::move(metadata_io)) +{ + regex_bag_pattern_ = R"(.+_(\d+)\.([a-zA-Z0-9])+)"; +} + +/// Determine which path should be placed first in a vector ordered by file number. +/** + * Used to re-order discovered bag files, since the filesystem discovery functions + * don't guarantee a preserved order + */ +bool Reindexer::compare_relative_file( + const rcpputils::fs::path & first_path, + const rcpputils::fs::path & second_path) +{ + std::regex regex_rule(regex_bag_pattern_, std::regex_constants::ECMAScript); + + std::smatch first_match; + std::smatch second_match; + + auto first_path_string = first_path.string(); + auto second_path_string = second_path.string(); + + auto first_regex_good = std::regex_match(first_path_string, first_match, regex_rule); + auto second_regex_good = std::regex_match(second_path_string, second_match, regex_rule); + + if (!first_regex_good) { + std::stringstream ss; + ss << "Path " << first_path.string() << + "didn't meet expected naming convention: " << regex_bag_pattern_; + std::string error_text = ss.str(); + throw std::runtime_error(error_text.c_str()); + } else if (!second_regex_good) { + std::stringstream ss; + ss << "Path " << second_path.string() << + "didn't meet expected naming convention: " << regex_bag_pattern_; + std::string error_text = ss.str(); + throw std::runtime_error(error_text.c_str()); + } + + auto first_db_num = std::stoul(first_match.str(1), nullptr, 10); + auto second_db_num = std::stoul(second_match.str(1), nullptr, 10); + + return first_db_num < second_db_num; +} + +/// Retrieve bag storage files from the bag directory. +/** + * @param base_folder: The bag directory that contains all the bag storage files + * @param output: A vector to save the discovered files inside of + * The files will be `emplace_back`-ed on the passed vector + */ +void Reindexer::get_bag_files( + const rcpputils::fs::path & base_folder, + std::vector & output) +{ + std::regex regex_rule(regex_bag_pattern_, std::regex_constants::ECMAScript); + auto allocator = rcutils_get_default_allocator(); + auto dir_iter = rcutils_dir_iter_start(base_folder.string().c_str(), allocator); + + // Make sure there are files in the directory + if (dir_iter == nullptr) { + throw std::runtime_error("Empty directory."); + } + + // Get all file names in directory + do { + auto found_file = rcpputils::fs::path(dir_iter->entry_name); + // Foxy compatibility. Adapts for https://github.com/ros2/rcpputils/pull/119 + rcpputils::fs::path base{base_folder_}; + ROSBAG2_CPP_LOG_DEBUG_STREAM("Found file: " << found_file.string()); + + if (std::regex_match(found_file.string(), regex_rule)) { + auto full_path = base / found_file; + output.emplace_back(full_path); + } + } while (rcutils_dir_iter_next(dir_iter)); + + // Sort relative file path by database number + std::sort( + output.begin(), output.end(), + [&, this](rcpputils::fs::path a, rcpputils::fs::path b) { + return compare_relative_file(a, b); + }); +} + +/// Prepare a fresh BagMetadata object for reindexing. +/** + * Creates a new `BagMetadata` object with the `storage_identifier` and `relative_file_paths` filled in + * Also fills in `starting_time` with a dummy default value. Important for later functions + */ +void Reindexer::init_metadata( + const std::vector & files, + const rosbag2_storage::StorageOptions & storage_options) +{ + metadata_ = rosbag2_storage::BagMetadata{}; + + metadata_.storage_identifier = storage_options.storage_id; + metadata_.starting_time = std::chrono::time_point( + std::chrono::nanoseconds::max()); + + // Record the relative paths to the metadata + for (const auto & path : files) { + auto cleaned_path = path.filename().string(); + metadata_.relative_file_paths.push_back(cleaned_path); + } +} + +/// Iterate through the bag files to collect various metadata parameters +/** + * Collects the topic metadata, `starting_time`, and `duration` portions of the `BagMetadata` + * being constructed + * @param: files The list of bag files to reindex + * @param: storage_options Used to construct the `Reader` needed to parse the bag files + */ +void Reindexer::aggregate_metadata( + const std::vector & files, + const std::unique_ptr & bag_reader, + const rosbag2_storage::StorageOptions & storage_options) +{ + std::map temp_topic_info; + + // In order to most accurately reconstruct the metadata, we need to + // visit each of the contained relative files files in the bag, + // open them, read the info, and write it into an aggregated metadata object. + ROSBAG2_CPP_LOG_DEBUG_STREAM("Extracting metadata from database(s)"); + for (const auto & f_ : files) { + ROSBAG2_CPP_LOG_DEBUG_STREAM("Extracting from file: " + f_.string()); + + metadata_.bag_size += f_.file_size(); + + // Set up reader + rosbag2_storage::StorageOptions temp_so = { + f_.string(), + storage_options.storage_id, + storage_options.max_bagfile_size, + storage_options.max_bagfile_duration, + storage_options.max_cache_size, + storage_options.storage_config_uri + }; + + // We aren't actually interested in reading messages, so use a blank converter option + rosbag2_cpp::ConverterOptions blank_converter_options {}; + bag_reader->open(temp_so, blank_converter_options); + auto temp_metadata = bag_reader->get_metadata(); + + if (temp_metadata.starting_time < metadata_.starting_time) { + metadata_.starting_time = temp_metadata.starting_time; + } + metadata_.duration += temp_metadata.duration; + ROSBAG2_CPP_LOG_DEBUG_STREAM("New duration: " + std::to_string(metadata_.duration.count())); + metadata_.message_count += temp_metadata.message_count; + + // Add the topic metadata + for (const auto & topic : temp_metadata.topics_with_message_count) { + auto found_topic = temp_topic_info.find(topic.topic_metadata.name); + if (found_topic == temp_topic_info.end()) { + // It's a new topic. Add it. + temp_topic_info[topic.topic_metadata.name] = topic; + } else { + ROSBAG2_CPP_LOG_DEBUG_STREAM("Found topic!"); + // Merge in the new information + found_topic->second.message_count += topic.message_count; + if (topic.topic_metadata.offered_qos_profiles != "") { + found_topic->second.topic_metadata.offered_qos_profiles = + topic.topic_metadata.offered_qos_profiles; + } + if (topic.topic_metadata.serialization_format != "") { + found_topic->second.topic_metadata.serialization_format = + topic.topic_metadata.serialization_format; + } + if (topic.topic_metadata.type != "") { + found_topic->second.topic_metadata.type = topic.topic_metadata.type; + } + } + } + + bag_reader->reset(); + } + + // Convert the topic map into topic metadata + for (auto & topic : temp_topic_info) { + metadata_.topics_with_message_count.emplace_back(topic.second); + } +} + +/// Reconstruct a bag's `metadata.yaml` file from the enclosed bag files. +/** + * The reindexer opens the files within the bag directory and uses the metadata of the files to + * reconstruct the metadata file. Currently does not support compressed bags. + * @param: storage_options The best-guess original storage options for the bag + */ +void Reindexer::reindex(const rosbag2_storage::StorageOptions & storage_options) +{ + base_folder_ = storage_options.uri; + ROSBAG2_CPP_LOG_INFO_STREAM("Beginning reindexing bag in directory: " << base_folder_.string()); + + auto metadata_io_default = std::make_unique(); + auto bag_reader = std::make_unique( + std::move(storage_factory_), converter_factory_, std::move(metadata_io_default)); + + // Identify all bag files + std::vector files; + get_bag_files(base_folder_, files); + if (files.empty()) { + throw std::runtime_error("No database files found for reindexing. Abort"); + } + + init_metadata(files, storage_options); + ROSBAG2_CPP_LOG_DEBUG_STREAM("Completed init_metadata"); + + // Collect all metadata from database files + aggregate_metadata(files, bag_reader, storage_options); + ROSBAG2_CPP_LOG_DEBUG_STREAM("Completed aggregate_metadata"); + + metadata_io_->write_metadata(base_folder_.string(), metadata_); + ROSBAG2_CPP_LOG_INFO("Reindexing complete."); +} +} // namespace rosbag2_cpp diff --git a/rosbag2_cpp/test/rosbag2_cpp/mock_storage.hpp b/rosbag2_cpp/test/rosbag2_cpp/mock_storage.hpp index 84bbf47ac3..303ddf9907 100644 --- a/rosbag2_cpp/test/rosbag2_cpp/mock_storage.hpp +++ b/rosbag2_cpp/test/rosbag2_cpp/mock_storage.hpp @@ -49,6 +49,7 @@ class MockStorage : public rosbag2_storage::storage_interfaces::ReadWriteInterfa MOCK_CONST_METHOD0(get_bagfile_size, uint64_t()); MOCK_CONST_METHOD0(get_relative_file_path, std::string()); MOCK_CONST_METHOD0(get_storage_identifier, std::string()); + MOCK_CONST_METHOD0(get_storage_extension, std::string()); MOCK_CONST_METHOD0(get_minimum_split_file_size, uint64_t()); }; diff --git a/rosbag2_py/CMakeLists.txt b/rosbag2_py/CMakeLists.txt index d354a97044..aad6ab41f6 100644 --- a/rosbag2_py/CMakeLists.txt +++ b/rosbag2_py/CMakeLists.txt @@ -125,6 +125,15 @@ reorder_pybind_include_directories(_info ) clean_windows_flags(_info) +pybind11_add_module(_reindexer SHARED + src/rosbag2_py/_reindexer.cpp +) +ament_target_dependencies(_reindexer PUBLIC + "rosbag2_cpp" + "rosbag2_storage" +) +clean_windows_flags(_reindexer) + # Install cython modules as sub-modules of the project install( TARGETS @@ -132,6 +141,7 @@ install( _storage _writer _info + _reindexer DESTINATION "${PYTHON_INSTALL_DIR}/${PROJECT_NAME}" ) @@ -161,6 +171,11 @@ if(BUILD_TESTING) PYTHON_EXECUTABLE "${_PYTHON_EXECUTABLE}" APPEND_ENV "PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}" ${other_environment_vars} ) + ament_add_pytest_test(test_reindexer_py + "test/test_reindexer.py" + PYTHON_EXECUTABLE "${_PYTHON_EXECUTABLE}" + APPEND_ENV "PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}" ${other_environment_vars} + ) endif() ament_package() diff --git a/rosbag2_py/resources/reindex_test_bags/multiple_files/multiple_files_0.db3 b/rosbag2_py/resources/reindex_test_bags/multiple_files/multiple_files_0.db3 new file mode 100644 index 0000000000..3edf751223 Binary files /dev/null and b/rosbag2_py/resources/reindex_test_bags/multiple_files/multiple_files_0.db3 differ diff --git a/rosbag2_py/resources/reindex_test_bags/multiple_files/multiple_files_1.db3 b/rosbag2_py/resources/reindex_test_bags/multiple_files/multiple_files_1.db3 new file mode 100644 index 0000000000..f367b03ac9 Binary files /dev/null and b/rosbag2_py/resources/reindex_test_bags/multiple_files/multiple_files_1.db3 differ diff --git a/rosbag2_py/resources/reindex_test_bags/multiple_files/multiple_files_2.db3 b/rosbag2_py/resources/reindex_test_bags/multiple_files/multiple_files_2.db3 new file mode 100644 index 0000000000..81735ad374 Binary files /dev/null and b/rosbag2_py/resources/reindex_test_bags/multiple_files/multiple_files_2.db3 differ diff --git a/rosbag2_py/rosbag2_py/__init__.py b/rosbag2_py/rosbag2_py/__init__.py index f4b1591cca..0522a95fd6 100644 --- a/rosbag2_py/rosbag2_py/__init__.py +++ b/rosbag2_py/rosbag2_py/__init__.py @@ -39,11 +39,15 @@ from rosbag2_py._info import ( Info, ) + from rosbag2_py._reindexer import ( + Reindexer + ) __all__ = [ 'ConverterOptions', 'get_registered_readers', 'get_registered_writers', + 'Reindexer', 'SequentialCompressionReader', 'SequentialCompressionWriter', 'SequentialReader', diff --git a/rosbag2_py/src/rosbag2_py/_reindexer.cpp b/rosbag2_py/src/rosbag2_py/_reindexer.cpp new file mode 100644 index 0000000000..97857a096f --- /dev/null +++ b/rosbag2_py/src/rosbag2_py/_reindexer.cpp @@ -0,0 +1,60 @@ +// Copyright 2021 DCS Corporation, All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// DISTRIBUTION A. Approved for public release; distribution unlimited. +// OPSEC #4584. +// +// Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS +// Part 252.227-7013 or 7014 (Feb 2014). +// +// This notice must appear in all copies of this file and its derivatives. + +#include +#include +#include + +#include "rosbag2_cpp/reindexer.hpp" +#include "rosbag2_storage/storage_options.hpp" + +#include "./pybind11.hpp" + +namespace rosbag2_py +{ + +class Reindexer +{ +public: + Reindexer() + : reindexer_(std::make_unique()) + { + } + + void reindex(const rosbag2_storage::StorageOptions & storage_options) + { + reindexer_->reindex(storage_options); + } + +protected: + std::unique_ptr reindexer_; +}; +} // namespace rosbag2_py + +PYBIND11_MODULE(_reindexer, m) { + m.doc() = "Python wrapper of the rosbag2_cpp reindexer API"; + + pybind11::class_( + m, "Reindexer") + .def(pybind11::init()) + .def("reindex", &rosbag2_py::Reindexer::reindex); +} diff --git a/rosbag2_py/test/test_reindexer.py b/rosbag2_py/test/test_reindexer.py new file mode 100644 index 0000000000..296fb4e367 --- /dev/null +++ b/rosbag2_py/test/test_reindexer.py @@ -0,0 +1,48 @@ +# Copyright 2021 DCS Corporation, All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DISTRIBUTION A. Approved for public release; distribution unlimited. +# OPSEC #4584. +# +# Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS +# Part 252.227-7013 or 7014 (Feb 2014). +# +# This notice must appear in all copies of this file and its derivatives. + +import os +from pathlib import Path +import sys + +if os.environ.get('ROSBAG2_PY_TEST_WITH_RTLD_GLOBAL', None) is not None: + # This is needed on Linux when compiling with clang/libc++. + # TL;DR This makes class_loader work when using a python extension compiled with libc++. + # + # For the fun RTTI ABI details, see https://whatofhow.wordpress.com/2015/03/17/odr-rtti-dso/. + sys.setdlopenflags(os.RTLD_GLOBAL | os.RTLD_LAZY) + +from common import get_rosbag_options # noqa +import rosbag2_py # noqa + + +def test_reindexer_multiple_files(): + bag_path = Path(__file__).parent.parent / 'resources' / 'reindex_test_bags' / 'multiple_files' + result_path = bag_path / 'metadata.yaml' + + storage_options, converter_options = get_rosbag_options(str(bag_path)) + reindexer = rosbag2_py.Reindexer() + reindexer.reindex(storage_options) + + assert(result_path.exists()) + + result_path.unlink(missing_ok=True) diff --git a/rosbag2_storage/test/rosbag2_storage/test_constants.hpp b/rosbag2_storage/test/rosbag2_storage/test_constants.hpp index d7a2ebc3ef..af91e80dd3 100644 --- a/rosbag2_storage/test/rosbag2_storage/test_constants.hpp +++ b/rosbag2_storage/test/rosbag2_storage/test_constants.hpp @@ -18,6 +18,8 @@ namespace test_constants { +constexpr const char * const READ_WRITE_PLUGIN_EXTENSION = ".rwplugin"; +constexpr const char * const READ_ONLY_PLUGIN_EXTENSION = ".roplugin"; constexpr const char * const READ_WRITE_PLUGIN_IDENTIFIER = "ReadWritePlugin"; constexpr const char * const READ_ONLY_PLUGIN_IDENTIFIER = "ReadOnlyPlugin"; constexpr const char * const DUMMY_FILEPATH = "/path/to/storage"; diff --git a/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp b/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp index 1af8a405ce..20063af55a 100644 --- a/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp +++ b/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp @@ -440,19 +440,19 @@ rosbag2_storage::BagMetadata SqliteStorage::get_metadata() auto statement = database_->prepare_statement( "SELECT name, type, serialization_format, COUNT(messages.id), MIN(messages.timestamp), " - "MAX(messages.timestamp) " + "MAX(messages.timestamp), offered_qos_profiles " "FROM messages JOIN topics on topics.id = messages.topic_id " "GROUP BY topics.name;"); auto query_results = statement->execute_query< std::string, std::string, std::string, int, rcutils_time_point_value_t, - rcutils_time_point_value_t>(); + rcutils_time_point_value_t, std::string>(); rcutils_time_point_value_t min_time = INT64_MAX; rcutils_time_point_value_t max_time = 0; for (auto result : query_results) { metadata.topics_with_message_count.push_back( { - {std::get<0>(result), std::get<1>(result), std::get<2>(result), ""}, + {std::get<0>(result), std::get<1>(result), std::get<2>(result), std::get<6>(result)}, static_cast(std::get<3>(result)) }); diff --git a/rosbag2_tests/CMakeLists.txt b/rosbag2_tests/CMakeLists.txt index 3f31e8e378..5530884f8e 100644 --- a/rosbag2_tests/CMakeLists.txt +++ b/rosbag2_tests/CMakeLists.txt @@ -97,6 +97,18 @@ if(BUILD_TESTING) rosbag2_storage) endif() + ament_add_gmock(test_reindex + test/rosbag2_tests/test_reindexer.cpp) + if(TARGET test_reindex) + ament_target_dependencies(test_reindex + rclcpp + rosbag2_cpp + rosbag2_storage + rosbag2_storage_default_plugins + rosbag2_test_common) + endif() + + ament_add_gmock(test_rosbag2_cpp_api test/rosbag2_tests/test_rosbag2_cpp_api.cpp WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_0.db3.zstd b/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_0.db3.zstd new file mode 100644 index 0000000000..47efd7e928 Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_0.db3.zstd differ diff --git a/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_1.db3.zstd b/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_1.db3.zstd new file mode 100644 index 0000000000..46a9604a27 Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_1.db3.zstd differ diff --git a/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_2.db3.zstd b/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_2.db3.zstd new file mode 100644 index 0000000000..f901849587 Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_2.db3.zstd differ diff --git a/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_3.db3.zstd b/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_3.db3.zstd new file mode 100644 index 0000000000..53c962965d Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_3.db3.zstd differ diff --git a/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_0.db3 b/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_0.db3 new file mode 100644 index 0000000000..cb496d68a3 Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_0.db3 differ diff --git a/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_1.db3 b/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_1.db3 new file mode 100644 index 0000000000..fe3aba22e7 Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_1.db3 differ diff --git a/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_2.db3 b/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_2.db3 new file mode 100644 index 0000000000..11a87eb30c Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_2.db3 differ diff --git a/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_3.db3 b/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_3.db3 new file mode 100644 index 0000000000..9b189dfa13 Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_3.db3 differ diff --git a/rosbag2_tests/resources/reindex_test_bags/multiple_files/multiple_files_0.db3 b/rosbag2_tests/resources/reindex_test_bags/multiple_files/multiple_files_0.db3 new file mode 100644 index 0000000000..3edf751223 Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/multiple_files/multiple_files_0.db3 differ diff --git a/rosbag2_tests/resources/reindex_test_bags/multiple_files/multiple_files_1.db3 b/rosbag2_tests/resources/reindex_test_bags/multiple_files/multiple_files_1.db3 new file mode 100644 index 0000000000..f367b03ac9 Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/multiple_files/multiple_files_1.db3 differ diff --git a/rosbag2_tests/resources/reindex_test_bags/multiple_files/multiple_files_2.db3 b/rosbag2_tests/resources/reindex_test_bags/multiple_files/multiple_files_2.db3 new file mode 100644 index 0000000000..81735ad374 Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/multiple_files/multiple_files_2.db3 differ diff --git a/rosbag2_tests/resources/reindex_test_bags/target_metadata/file_compression/metadata.yaml b/rosbag2_tests/resources/reindex_test_bags/target_metadata/file_compression/metadata.yaml new file mode 100644 index 0000000000..764e22a4c1 --- /dev/null +++ b/rosbag2_tests/resources/reindex_test_bags/target_metadata/file_compression/metadata.yaml @@ -0,0 +1,34 @@ +rosbag2_bagfile_information: + version: 4 + storage_identifier: sqlite3 + relative_file_paths: + - file_compression/file_compression_0.db3.zstd + - file_compression/file_compression_1.db3.zstd + - file_compression/file_compression_2.db3.zstd + - file_compression/file_compression_3.db3.zstd + duration: + nanoseconds: 59997866716 + starting_time: + nanoseconds_since_epoch: 1604357089919704830 + message_count: 775 + topics_with_message_count: + - topic_metadata: + name: /topic + type: std_msgs/msg/String + serialization_format: cdr + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 2147483647\n nsec: 4294967295\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + message_count: 383 + - topic_metadata: + name: /parameter_events + type: rcl_interfaces/msg/ParameterEvent + serialization_format: cdr + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 2147483647\n nsec: 4294967295\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + message_count: 0 + - topic_metadata: + name: /rosout + type: rcl_interfaces/msg/Log + serialization_format: cdr + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 1\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 10\n nsec: 0\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + message_count: 392 + compression_format: zstd + compression_mode: FILE \ No newline at end of file diff --git a/rosbag2_tests/resources/reindex_test_bags/target_metadata/message_compression/metadata.yaml b/rosbag2_tests/resources/reindex_test_bags/target_metadata/message_compression/metadata.yaml new file mode 100644 index 0000000000..147265e2c7 --- /dev/null +++ b/rosbag2_tests/resources/reindex_test_bags/target_metadata/message_compression/metadata.yaml @@ -0,0 +1,34 @@ +rosbag2_bagfile_information: + version: 4 + storage_identifier: sqlite3 + relative_file_paths: + - message_compression/message_compression_0.db3 + - message_compression/message_compression_1.db3 + - message_compression/message_compression_2.db3 + - message_compression/message_compression_3.db3 + duration: + nanoseconds: 59872587605 + starting_time: + nanoseconds_since_epoch: 1604581538133709023 + message_count: 1127 + topics_with_message_count: + - topic_metadata: + name: /parameter_events + type: rcl_interfaces/msg/ParameterEvent + serialization_format: cdr + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 2147483647\n nsec: 4294967295\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false\n- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 2147483647\n nsec: 4294967295\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + message_count: 0 + - topic_metadata: + name: /rosout + type: rcl_interfaces/msg/Log + serialization_format: cdr + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 1\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 10\n nsec: 0\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false\n- history: 3\n depth: 0\n reliability: 1\n durability: 1\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 10\n nsec: 0\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + message_count: 758 + - topic_metadata: + name: /topic + type: std_msgs/msg/String + serialization_format: cdr + offered_qos_profiles: "" + message_count: 369 + compression_format: zstd + compression_mode: MESSAGE \ No newline at end of file diff --git a/rosbag2_tests/resources/reindex_test_bags/target_metadata/multiple_files/metadata.yaml b/rosbag2_tests/resources/reindex_test_bags/target_metadata/multiple_files/metadata.yaml new file mode 100644 index 0000000000..6986d37388 --- /dev/null +++ b/rosbag2_tests/resources/reindex_test_bags/target_metadata/multiple_files/metadata.yaml @@ -0,0 +1,21 @@ +rosbag2_bagfile_information: + version: 4 + storage_identifier: sqlite3 + relative_file_paths: + - multiple_files_0.db3 + - multiple_files_1.db3 + - multiple_files_2.db3 + duration: + nanoseconds: 1616630880179528024 + starting_time: + nanoseconds_since_epoch: 22460796745086 + message_count: 3177 + topics_with_message_count: + - topic_metadata: + name: /chatter + type: std_msgs/msg/String + serialization_format: cdr + offered_qos_profiles: "- history: 1\n depth: 1\n reliability: 1\n durability: 2\n deadline:\n sec: 9223372036\n nsec: 854775807\n lifespan:\n sec: 9223372036\n nsec: 854775807\n liveliness: 1\n liveliness_lease_duration:\n sec: 9223372036\n nsec: 854775807\n avoid_ros_namespace_conventions: false" + message_count: 3177 + compression_format: "" + compression_mode: "" \ No newline at end of file diff --git a/rosbag2_tests/test/rosbag2_tests/test_reindexer.cpp b/rosbag2_tests/test/rosbag2_tests/test_reindexer.cpp new file mode 100644 index 0000000000..806891d722 --- /dev/null +++ b/rosbag2_tests/test/rosbag2_tests/test_reindexer.cpp @@ -0,0 +1,113 @@ +// Copyright 2021 DCS Corporation, All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// DISTRIBUTION A. Approved for public release; distribution unlimited. +// OPSEC #4584. +// +// Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS +// Part 252.227-7013 or 7014 (Feb 2014). +// +// This notice must appear in all copies of this file and its derivatives. + +#include + +#include +#include +#include +#include +#include + +#include "rcpputils/asserts.hpp" +#include "rcpputils/filesystem_helper.hpp" + +#include "rosbag2_cpp/readers/sequential_reader.hpp" +#include "rosbag2_cpp/reindexer.hpp" + +#include "rosbag2_storage/bag_metadata.hpp" +#include "rosbag2_storage/metadata_io.hpp" +#include "rosbag2_storage/topic_metadata.hpp" + +using namespace testing; // NOLINT + +class ReindexTestFixture : public Test +{ +public: + ReindexTestFixture() + { + database_path = rcpputils::fs::path(_SRC_RESOURCES_DIR_PATH) / "reindex_test_bags"; + target_dir = database_path / "target_metadata"; + } + + rcpputils::fs::path database_path; + rcpputils::fs::path target_dir; +}; + +TEST_F(ReindexTestFixture, test_multiple_files) { + auto bag_dir = database_path / "multiple_files"; + std::unique_ptr reindexer = + std::make_unique(); + + rosbag2_storage::StorageOptions so = rosbag2_storage::StorageOptions(); + so.uri = bag_dir.string(); + so.storage_id = "sqlite3"; + + reindexer->reindex(so); + + auto generated_file = rcpputils::fs::path(bag_dir) / "metadata.yaml"; + EXPECT_TRUE(generated_file.exists()); + + auto metadata_io = std::make_unique(); + auto generated_metadata = metadata_io->read_metadata(bag_dir.string()); + auto target_metadata = metadata_io->read_metadata((target_dir / "multiple_files").string()); + + EXPECT_EQ(generated_metadata.version, target_metadata.version); + + for (const auto & gen_rel_path : generated_metadata.relative_file_paths) { + EXPECT_TRUE( + std::find( + target_metadata.relative_file_paths.begin(), + target_metadata.relative_file_paths.end(), + gen_rel_path) != target_metadata.relative_file_paths.end()); + } + + // Disabled for now, since it may not be possible to 100% recreate + // original starting time from metadata + // EXPECT_EQ(generated_metadata.starting_time, target_metadata.starting_time); + + // Disabled for now, since I'm not sure how duration is created, and if it's correct (jhdcs) + // EXPECT_EQ(generated_metadata.duration, target_metadata.duration); + + EXPECT_EQ(generated_metadata.message_count, target_metadata.message_count); + + // Reindexer can only reconstruct topics that had messages, so not all topics may exist + for (const auto & gen_topic : generated_metadata.topics_with_message_count) { + EXPECT_TRUE( + std::find_if( + target_metadata.topics_with_message_count.begin(), + target_metadata.topics_with_message_count.end(), + [&gen_topic](rosbag2_storage::TopicInformation & t) { + return (t.topic_metadata.name == gen_topic.topic_metadata.name) && + (t.message_count == gen_topic.message_count) && + (t.topic_metadata.offered_qos_profiles == + gen_topic.topic_metadata.offered_qos_profiles) && + (t.topic_metadata.type == gen_topic.topic_metadata.type); + } + ) != target_metadata.topics_with_message_count.end() + ); + } + + if (generated_file.exists()) { + remove(generated_file); + } +}