feat(clp-s): Add the read-side implementation for the archive range index.

components/core/CMakeLists.txt

@@ -633,6 +633,7 @@ set(SOURCE_FILES_unitTest
         tests/test-BoundedReader.cpp
         tests/test-BufferedFileReader.cpp
         tests/test-clp_s-end_to_end.cpp
+        tests/test-clp_s-range_index.cpp
         tests/test-clp_s-search.cpp
         tests/test-EncodedVariableInterpreter.cpp
         tests/test-encoding_methods.cpp

components/core/src/clp_s/ArchiveReader.hpp

@@ -117,6 +117,10 @@ class ArchiveReader {
 
     std::shared_ptr<ReaderUtils::SchemaMap> get_schema_map() { return m_schema_map; }
 
+    auto get_range_index() const -> std::vector<ArchiveReaderAdaptor::RangeIndexEntry> const& {
+        return m_archive_reader_adaptor->get_range_index();
+    }
+
     /**
      * Writes decoded messages to a file.
      * @param writer

components/core/src/clp_s/ArchiveReaderAdaptor.cpp

@@ -6,15 +6,18 @@
 #include <optional>
 #include <string>
 #include <string_view>
+#include <utility>
 #include <vector>
 
 #include <msgpack.hpp>
+#include <nlohmann/json.hpp>
 #include <spdlog/spdlog.h>
 
 #include "../clp/BoundedReader.hpp"
 #include "../clp/FileReader.hpp"
 #include "archive_constants.hpp"
 #include "InputConfig.hpp"
+#include "RangeIndexWriter.hpp"
 #include "ReaderUtils.hpp"
 #include "SingleFileArchiveDefs.hpp"
 
@@ -97,6 +100,54 @@ ErrorCode ArchiveReaderAdaptor::try_read_archive_info(ZstdDecompressor& decompre
     return ErrorCodeSuccess;
 }
 
+auto ArchiveReaderAdaptor::try_read_range_index(ZstdDecompressor& decompressor, size_t size)
+        -> ErrorCode {
+    std::vector<char> buffer(size);
+    if (auto const rc = decompressor.try_read_exact_length(buffer.data(), buffer.size());
+        ErrorCodeSuccess != rc)
+    {
+        return rc;
+    }
+
+    auto range_index_json = nlohmann::json::from_msgpack(buffer.begin(), buffer.end(), true, false);
+    if (false == range_index_json.is_array()) {
+        return ErrorCodeCorrupt;
+    }
+
+    for (auto& range_index_entry : range_index_json) {
+        if (false == range_index_entry.contains(RangeIndexWriter::cStartIndexName)
+            || false == range_index_entry.at(RangeIndexWriter::cStartIndexName).is_number_integer())
+        {
+            return ErrorCodeCorrupt;
+        }
+        if (false == range_index_entry.contains(RangeIndexWriter::cEndIndexName)
+            || false == range_index_entry.at(RangeIndexWriter::cEndIndexName).is_number_integer())
+        {
+            return ErrorCodeCorrupt;
+        }
+        if (false == range_index_entry.contains(RangeIndexWriter::cMetadataFieldsName)
+            || false == range_index_entry.at(RangeIndexWriter::cMetadataFieldsName).is_object())
+        {
+            return ErrorCodeCorrupt;
+        }
+        auto start_index{
+                range_index_entry.at(RangeIndexWriter::cStartIndexName).template get<size_t>()
+        };
+        auto end_index{
+                range_index_entry.at(RangeIndexWriter::cEndIndexName).template get<size_t>()
+        };
+        if (start_index > end_index) {
+            return ErrorCodeCorrupt;
+        }
+        m_range_index.emplace_back(
+                start_index,
+                end_index,
+                std::move(range_index_entry.at(RangeIndexWriter::cMetadataFieldsName))
+        );
+    }
+    return ErrorCodeSuccess;
+}
+
 auto
 ArchiveReaderAdaptor::try_read_unknown_metadata_packet(ZstdDecompressor& decompressor, size_t size)
         -> ErrorCode {
@@ -181,6 +232,9 @@ ErrorCode ArchiveReaderAdaptor::try_read_archive_metadata(ZstdDecompressor& deco
             case ArchiveMetadataPacketType::ArchiveInfo:
                 rc = try_read_archive_info(decompressor, packet_size);
                 break;
+            case ArchiveMetadataPacketType::RangeIndex:
+                rc = try_read_range_index(decompressor, packet_size);
+                break;
             default:
                 rc = try_read_unknown_metadata_packet(decompressor, packet_size);
                 break;

components/core/src/clp_s/ArchiveReaderAdaptor.hpp

@@ -1,10 +1,20 @@
 #ifndef CLP_S_ARCHIVEREADERADAPTOR_HPP
 #define CLP_S_ARCHIVEREADERADAPTOR_HPP
 
+#include <cstddef>
 #include <memory>
 #include <optional>
 #include <string>
 #include <string_view>
+#include <utility>
+#include <vector>
+
+// We use NOLINTNEXTLINE to satisfy clang-tidy here because while we don't use any symbols from
+// `nlohmann/json.hpp` directly this code does not compile without the definition of
+// `nlohmann::basic_json<>` found in the `nlohmann/json.hpp` header.
+// NOLINTNEXTLINE(misc-include-cleaner)
+#include <nlohmann/json.hpp>
+#include <nlohmann/json_fwd.hpp>
 
 #include "../clp/BoundedReader.hpp"
 #include "../clp/ReaderInterface.hpp"
@@ -28,6 +38,18 @@ class ArchiveReaderAdaptor {
                 : TraceableException(error_code, filename, line_number) {}
     };
 
+    struct RangeIndexEntry {
+        explicit RangeIndexEntry(size_t start_index, size_t end_index, nlohmann::json&& fields)
+                : start_index{start_index},
+                  end_index{end_index},
+                  // Note: brace initializer would make nlohmann wrap the fields object in an array.
+                  fields(std::move(fields)) {}
+
+        size_t start_index;
+        size_t end_index;
+        nlohmann::json fields;
+    };
+
     explicit ArchiveReaderAdaptor(Path const& archive_path, NetworkAuthOption const& network_auth);
 
     /**
@@ -62,6 +84,8 @@ class ArchiveReaderAdaptor {
 
     ArchiveHeader const& get_header() const { return m_archive_header; }
 
+    std::vector<RangeIndexEntry> const& get_range_index() const { return m_range_index; }
+
 private:
     /**
      * Tries to read an ArchiveFileInfo packet from the archive metadata.
@@ -90,6 +114,14 @@ class ArchiveReaderAdaptor {
      */
     ErrorCode try_read_archive_info(ZstdDecompressor& decompressor, size_t size);
 
+    /**
+     * Tries to read a RangeIndex packet from the archive metadata.
+     * @param decompressor
+     * @param size The number of decompressed bytes making up the packet.
+     * @return ErrorCodeSuccess on success or the relevant ErrorCode on failure.
+     */
+    auto try_read_range_index(ZstdDecompressor& decompressor, size_t size) -> ErrorCode;
+
     /**
      * Tries to read an unknown metadata packet from the archive metadata.
      * @param decompressor
@@ -140,6 +172,7 @@ class ArchiveReaderAdaptor {
     std::optional<std::string> m_current_reader_holder;
     std::shared_ptr<TimestampDictionaryReader> m_timestamp_dictionary;
     std::shared_ptr<clp::ReaderInterface> m_reader;
+    std::vector<RangeIndexEntry> m_range_index;
 };
 }  // namespace clp_s
 #endif  // CLP_S_ARCHIVEREADERADAPTOR_HPP

components/core/src/clp_s/indexer/CMakeLists.txt

@@ -90,6 +90,7 @@ target_link_libraries(indexer
         clp::string_utils
         date::date
         MariaDBClient::MariaDBClient
+        nlohmann_json::nlohmann_json
         OpenSSL::Crypto
         simdjson::simdjson
         spdlog::spdlog