Split StreamReader into an interface and implementation to prepare for adding another IR stream reader.

src/clp_ffi_js/ir/IrStreamReader.cpp

@@ -0,0 +1,264 @@
+#include "IrStreamReader.hpp"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <iterator>
+#include <memory>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <system_error>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include <clp/Array.hpp>
+#include <clp/ErrorCode.hpp>
+#include <clp/ir/LogEventDeserializer.hpp>
+#include <clp/ir/types.hpp>
+#include <clp/streaming_compression/zstd/Decompressor.hpp>
+#include <clp/TraceableException.hpp>
+#include <clp/type_utils.hpp>
+#include <emscripten/bind.h>
+#include <emscripten/em_asm.h>
+#include <emscripten/val.h>
+#include <spdlog/spdlog.h>
+
+#include <clp_ffi_js/ClpFfiJsException.hpp>
+#include <clp_ffi_js/constants.hpp>
+#include <clp_ffi_js/ir/decoding_methods.hpp>
+#include <clp_ffi_js/ir/LogEventWithLevel.hpp>
+#include <clp_ffi_js/ir/StreamReader.hpp>
+#include <clp_ffi_js/ir/StreamReaderDataContext.hpp>
+
+using namespace std::literals::string_literals;
+using clp::ir::four_byte_encoded_variable_t;
+
+namespace clp_ffi_js::ir {
+auto IrStreamReader::create(DataArrayTsType const& data_array) -> IrStreamReader {
+    auto const length{data_array["length"].as<size_t>()};
+    SPDLOG_INFO("IrStreamReader::create: got buffer of length={}", length);
+
+    // Copy array from JavaScript to C++
+    clp::Array<char> data_buffer{length};
+    // NOLINTBEGIN(cppcoreguidelines-pro-type-reinterpret-cast)
+    emscripten::val::module_property("HEAPU8")
+            .call<void>("set", data_array, reinterpret_cast<uintptr_t>(data_buffer.data()));
+    // NOLINTEND(cppcoreguidelines-pro-type-reinterpret-cast)
+
+    auto zstd_decompressor{std::make_unique<clp::streaming_compression::zstd::Decompressor>()};
+    zstd_decompressor->open(data_buffer.data(), length);
+
+    auto stream_reader_data_context{create_deserializer_and_data_context(
+            std::move(zstd_decompressor),
+            std::move(data_buffer)
+    )};
+    return IrStreamReader{std::move(stream_reader_data_context)};
+}
+
+auto IrStreamReader::get_num_events_buffered() const -> size_t {
+    return m_encoded_log_events.size();
+}
+
+auto IrStreamReader::get_filtered_log_event_map() const -> FilteredLogEventMapTsType {
+    if (false == m_filtered_log_event_map.has_value()) {
+        return FilteredLogEventMapTsType{emscripten::val::null()};
+    }
+
+    return FilteredLogEventMapTsType{emscripten::val::array(m_filtered_log_event_map.value())};
+}
+
+void IrStreamReader::filter_log_events(emscripten::val const& log_level_filter) {
+    if (log_level_filter.isNull()) {
+        m_filtered_log_event_map.reset();
+        return;
+    }
+
+    m_filtered_log_event_map.emplace();
+    auto filter_levels{emscripten::vecFromJSArray<std::underlying_type_t<LogLevel>>(log_level_filter
+    )};
+    for (size_t log_event_idx = 0; log_event_idx < m_encoded_log_events.size(); ++log_event_idx) {
+        auto const& log_event = m_encoded_log_events[log_event_idx];
+        if (std::ranges::find(
+                    filter_levels,
+                    clp::enum_to_underlying_type(log_event.get_log_level())
+            )
+            != filter_levels.end())
+        {
+            m_filtered_log_event_map->emplace_back(log_event_idx);
+        }
+    }
+}
+
+auto IrStreamReader::deserialize_stream() -> size_t {
+    if (nullptr == m_stream_reader_data_context) {
+        return m_encoded_log_events.size();
+    }
+
+    constexpr size_t cDefaultNumReservedLogEvents{500'000};
+    m_encoded_log_events.reserve(cDefaultNumReservedLogEvents);
+
+    while (true) {
+        auto result{m_stream_reader_data_context->get_deserializer().deserialize_log_event()};
+        if (result.has_error()) {
+            auto const error{result.error()};
+            if (std::errc::no_message_available == error) {
+                break;
+            }
+            if (std::errc::result_out_of_range == error) {
+                SPDLOG_ERROR("File contains an incomplete IR stream");
+                break;
+            }
+            throw ClpFfiJsException{
+                    clp::ErrorCode::ErrorCode_Corrupt,
+                    __FILENAME__,
+                    __LINE__,
+                    "Failed to deserialize: "s + error.category().name() + ":" + error.message()
+            };
+        }
+        auto const& log_event = result.value();
+        auto const& message = log_event.get_message();
+
+        auto const& logtype = message.get_logtype();
+        constexpr size_t cLogLevelPositionInMessages{1};
+        LogLevel log_level{LogLevel::NONE};
+        if (logtype.length() > cLogLevelPositionInMessages) {
+            // NOLINTNEXTLINE(readability-qualified-auto)
+            auto const log_level_name_it{std::find_if(
+                    cLogLevelNames.begin() + static_cast<size_t>(cValidLogLevelsBeginIdx),
+                    cLogLevelNames.end(),
+                    [&](std::string_view level) {
+                        return logtype.substr(cLogLevelPositionInMessages).starts_with(level);
+                    }
+            )};
+            if (log_level_name_it != cLogLevelNames.end()) {
+                log_level = static_cast<LogLevel>(
+                        std::distance(cLogLevelNames.begin(), log_level_name_it)
+                );
+            }
+        }
+
+        auto log_viewer_event{LogEventWithLevel<four_byte_encoded_variable_t>(
+                log_event.get_timestamp(),
+                log_event.get_utc_offset(),
+                message,
+                log_level
+        )};
+        m_encoded_log_events.emplace_back(std::move(log_viewer_event));
+    }
+    m_stream_reader_data_context.reset(nullptr);
+    return m_encoded_log_events.size();
+}
+
+auto IrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bool use_filter) const
+        -> DecodedResultsTsType {
+    if (use_filter && false == m_filtered_log_event_map.has_value()) {
+        return DecodedResultsTsType{emscripten::val::null()};
+    }
+
+    size_t length{0};
+    if (use_filter) {
+        length = m_filtered_log_event_map->size();
+    } else {
+        length = m_encoded_log_events.size();
+    }
+    if (length < end_idx || begin_idx > end_idx) {
+        return DecodedResultsTsType{emscripten::val::null()};
+    }
+
+    std::string message;
+    constexpr size_t cDefaultReservedMessageLength{512};
+    message.reserve(cDefaultReservedMessageLength);
+    auto const results{emscripten::val::array()};
+
+    for (size_t i = begin_idx; i < end_idx; ++i) {
+        size_t log_event_idx{0};
+        if (use_filter) {
+            log_event_idx = m_filtered_log_event_map->at(i);
+        } else {
+            log_event_idx = i;
+        }
+        auto const& log_event{m_encoded_log_events[log_event_idx]};
+
+        auto const parsed{log_event.get_message().decode_and_unparse()};
+        if (false == parsed.has_value()) {
+            SPDLOG_ERROR("Failed to decode message.");
+            break;
+        }
+        message = parsed.value();
+
+        m_ts_pattern.insert_formatted_timestamp(log_event.get_timestamp(), message);
+
+        EM_ASM(
+                { Emval.toValue($0).push([UTF8ToString($1), $2, $3, $4]); },
+                results.as_handle(),
+                message.c_str(),
+                log_event.get_timestamp(),
+                log_event.get_log_level(),
+                log_event_idx + 1
+        );
+    }
+
+    return DecodedResultsTsType(results);
+}
+
+IrStreamReader::IrStreamReader(
+        StreamReaderDataContext<four_byte_encoded_variable_t>&& stream_reader_data_context
+)
+        : m_stream_reader_data_context{std::make_unique<
+                  StreamReaderDataContext<four_byte_encoded_variable_t>>(
+                  std::move(stream_reader_data_context)
+          )},
+          m_ts_pattern{m_stream_reader_data_context->get_deserializer().get_timestamp_pattern()} {}
+
+auto IrStreamReader::create_data_context(
+        std::unique_ptr<clp::streaming_compression::zstd::Decompressor>&& zstd_decompressor,
+        clp::Array<char>&& data_buffer
+) -> StreamReaderDataContext<four_byte_encoded_variable_t> {
+    rewind_reader_and_validate_encoding_type(*zstd_decompressor);
+
+    auto result{
+            clp::ir::LogEventDeserializer<four_byte_encoded_variable_t>::create(*zstd_decompressor)
+    };
+    if (result.has_error()) {
+        auto const error_code{result.error()};
+        SPDLOG_CRITICAL(
+                "Failed to create deserializer: {}:{}",
+                error_code.category().name(),
+                error_code.message()
+        );
+        throw ClpFfiJsException{
+                clp::ErrorCode::ErrorCode_Failure,
+                __FILENAME__,
+                __LINE__,
+                "Failed to create deserializer"
+        };
+    }
+
+    return {std::move(data_buffer), std::move(zstd_decompressor), std::move(result.value())};
+}
+}  // namespace clp_ffi_js::ir
+
+namespace {
+EMSCRIPTEN_BINDINGS(ClpIrStreamReader) {
+    emscripten::class_<
+            clp_ffi_js::ir::IrStreamReader,
+            emscripten::base<clp_ffi_js::ir::StreamReader>>("ClpIrStreamReader")
+            .constructor(
+                    &clp_ffi_js::ir::IrStreamReader::create,
+                    emscripten::return_value_policy::take_ownership()
+            )
+            .function(
+                    "getNumEventsBuffered",
+                    &clp_ffi_js::ir::IrStreamReader::get_num_events_buffered
+            )
+            .function(
+                    "getFilteredLogEventMap",
+                    &clp_ffi_js::ir::IrStreamReader::get_filtered_log_event_map
+            )
+            .function("filterLogEvents", &clp_ffi_js::ir::IrStreamReader::filter_log_events)
+            .function("deserializeStream", &clp_ffi_js::ir::IrStreamReader::deserialize_stream)
+            .function("decodeRange", &clp_ffi_js::ir::IrStreamReader::decode_range);
+}
+}  // namespace

src/clp_ffi_js/ir/IrStreamReader.hpp

@@ -0,0 +1,122 @@
+#ifndef CLP_FFI_JS_IR_IR_STREAM_READER_HPP
+#define CLP_FFI_JS_IR_IR_STREAM_READER_HPP
+
+#include <Array.hpp>
+#include <cstddef>
+#include <memory>
+#include <optional>
+#include <streaming_compression/zstd/Decompressor.hpp>
+#include <vector>
+
+#include <clp/ir/types.hpp>
+#include <clp/TimestampPattern.hpp>
+#include <emscripten/val.h>
+
+#include <clp_ffi_js/ir/LogEventWithLevel.hpp>
+#include <clp_ffi_js/ir/StreamReader.hpp>
+#include <clp_ffi_js/ir/StreamReaderDataContext.hpp>
+
+using clp::ir::four_byte_encoded_variable_t;
+
+namespace clp_ffi_js::ir {
+/**
+ * Class to deserialize and decode Zstandard-compressed CLP IR streams as well as format decoded
+ * log events.
+ */
+class IrStreamReader : public StreamReader {
+    friend StreamReader;
+
+public:
+    /**
+     * Mapping between an index in the filtered log events collection to an index in the unfiltered
+     * log events collection.
+     */
+    using FilteredLogEventsMap = std::optional<std::vector<size_t>>;
+
+    /**
+     * Creates a IrStreamReader to read from the given array.
+     *
+     * @param data_array An array containing a Zstandard-compressed IR stream.
+     * @return The created instance.
+     * @throw ClpFfiJsException if any error occurs.
+     */
+    [[nodiscard]] static auto create(DataArrayTsType const& data_array) -> IrStreamReader;
+
+    // Destructor
+    ~IrStreamReader() override = default;
+
+    // Disable copy constructor and assignment operator
+    IrStreamReader(IrStreamReader const&) = delete;
+    auto operator=(IrStreamReader const&) -> IrStreamReader& = delete;
+
+    // Define default move constructor
+    IrStreamReader(IrStreamReader&&) = default;
+    // Delete move assignment operator since it's also disabled in `clp::ir::LogEventDeserializer`.
+    auto operator=(IrStreamReader&&) -> IrStreamReader& = delete;
+
+    // Methods
+    /**
+     * @return The number of events buffered.
+     */
+    [[nodiscard]] auto get_num_events_buffered() const -> size_t override;
+
+    /**
+     * @return The filtered log events map.
+     */
+    [[nodiscard]] auto get_filtered_log_event_map() const -> FilteredLogEventMapTsType override;
+
+    /**
+     * Generates a filtered collection from all log events.
+     *
+     * @param log_level_filter Array of selected log levels
+     */
+    void filter_log_events(emscripten::val const& log_level_filter) override;
+
+    /**
+     * Deserializes all log events in the stream. After the stream has been exhausted, it will be
+     * deallocated.
+     *
+     * @return The number of successfully deserialized ("valid") log events.
+     */
+    [[nodiscard]] auto deserialize_stream() -> size_t override;
+
+    /**
+     * Decodes log events in the range `[beginIdx, endIdx)` of the filtered or unfiltered
+     * (depending on the value of `useFilter`) log events collection.
+     *
+     * @param begin_idx
+     * @param end_idx
+     * @param use_filter Whether to decode from the filtered or unfiltered log events collection.
+     * @return An array where each element is a decoded log event represented by an array of:
+     * - The log event's message
+     * - The log event's timestamp as milliseconds since the Unix epoch
+     * - The log event's log level as an integer that indexes into `cLogLevelNames`
+     * - The log event's number (1-indexed) in the stream
+     * @return null if any log event in the range doesn't exist (e.g. the range exceeds the number
+     * of log events in the collection).
+     */
+    [[nodiscard]] auto decode_range(size_t begin_idx, size_t end_idx, bool use_filter) const
+            -> DecodedResultsTsType override;
+
+private:
+    // Constructor
+    explicit IrStreamReader(
+            StreamReaderDataContext<four_byte_encoded_variable_t>&& stream_reader_data_context
+    );
+
+    // Methods
+    [[nodiscard]] static auto create_deserializer_and_data_context(
+            std::unique_ptr<clp::streaming_compression::zstd::Decompressor>&& zstd_decompressor,
+            clp::Array<char>&& data_buffer
+    ) -> StreamReaderDataContext<four_byte_encoded_variable_t>;
+
+    // Variables
+    std::vector<LogEventWithLevel<four_byte_encoded_variable_t>> m_encoded_log_events;
+    std::unique_ptr<StreamReaderDataContext<four_byte_encoded_variable_t>>
+            m_stream_reader_data_context;
+    FilteredLogEventsMap m_filtered_log_event_map;
+    clp::TimestampPattern m_ts_pattern;
+};
+}  // namespace clp_ffi_js::ir
+
+#endif  // CLP_FFI_JS_IR_IR_STREAM_READER_HPP