diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 9846615984..d9f4ba24f7 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -630,11 +630,14 @@ set(SOURCE_FILES_unitTest src/clp/version.hpp src/clp/WriterInterface.cpp src/clp/WriterInterface.hpp + tests/clp_s_test_utils.cpp + tests/clp_s_test_utils.hpp tests/LogSuppressor.hpp tests/TestOutputCleaner.hpp tests/test-BoundedReader.cpp tests/test-BufferedFileReader.cpp tests/test-clp_s-end_to_end.cpp + tests/test-clp_s-range_index.cpp tests/test-clp_s-search.cpp tests/test-EncodedVariableInterpreter.cpp tests/test-encoding_methods.cpp diff --git a/components/core/src/clp_s/ArchiveReader.hpp b/components/core/src/clp_s/ArchiveReader.hpp index e9f47cd399..41a64cf503 100644 --- a/components/core/src/clp_s/ArchiveReader.hpp +++ b/components/core/src/clp_s/ArchiveReader.hpp @@ -117,6 +117,10 @@ class ArchiveReader { std::shared_ptr get_schema_map() { return m_schema_map; } + auto get_range_index() const -> std::vector const& { + return m_archive_reader_adaptor->get_range_index(); + } + /** * Writes decoded messages to a file. * @param writer diff --git a/components/core/src/clp_s/ArchiveReaderAdaptor.cpp b/components/core/src/clp_s/ArchiveReaderAdaptor.cpp index a7bbc80612..ba8dc262d0 100644 --- a/components/core/src/clp_s/ArchiveReaderAdaptor.cpp +++ b/components/core/src/clp_s/ArchiveReaderAdaptor.cpp @@ -6,15 +6,18 @@ #include #include #include +#include #include #include +#include #include #include "../clp/BoundedReader.hpp" #include "../clp/FileReader.hpp" #include "archive_constants.hpp" #include "InputConfig.hpp" +#include "RangeIndexWriter.hpp" #include "ReaderUtils.hpp" #include "SingleFileArchiveDefs.hpp" @@ -97,6 +100,58 @@ ErrorCode ArchiveReaderAdaptor::try_read_archive_info(ZstdDecompressor& decompre return ErrorCodeSuccess; } +auto ArchiveReaderAdaptor::try_read_range_index(ZstdDecompressor& decompressor, size_t size) + -> ErrorCode { + std::vector buffer(size); + if (auto const rc = decompressor.try_read_exact_length(buffer.data(), buffer.size()); + ErrorCodeSuccess != rc) + { + return rc; + } + + auto range_index_json = nlohmann::json::from_msgpack(buffer.begin(), buffer.end(), true, false); + if (false == range_index_json.is_array()) { + return ErrorCodeCorrupt; + } + + for (auto& range_index_entry : range_index_json) { + if (false == range_index_entry.contains(RangeIndexWriter::cStartIndexName) + || false == range_index_entry.at(RangeIndexWriter::cStartIndexName).is_number_integer()) + { + return ErrorCodeCorrupt; + } + if (false == range_index_entry.contains(RangeIndexWriter::cEndIndexName) + || false == range_index_entry.at(RangeIndexWriter::cEndIndexName).is_number_integer()) + { + return ErrorCodeCorrupt; + } + if (false == range_index_entry.contains(RangeIndexWriter::cMetadataFieldsName) + || false == range_index_entry.at(RangeIndexWriter::cMetadataFieldsName).is_object()) + { + return ErrorCodeCorrupt; + } + size_t start_index{}; + size_t end_index{}; + try { + start_index = range_index_entry.at(RangeIndexWriter::cStartIndexName) + .template get(); + end_index + = range_index_entry.at(RangeIndexWriter::cEndIndexName).template get(); + } catch (std::exception const&) { + return ErrorCodeCorrupt; + } + if (start_index > end_index) { + return ErrorCodeCorrupt; + } + m_range_index.emplace_back( + start_index, + end_index, + std::move(range_index_entry.at(RangeIndexWriter::cMetadataFieldsName)) + ); + } + return ErrorCodeSuccess; +} + auto ArchiveReaderAdaptor::try_read_unknown_metadata_packet(ZstdDecompressor& decompressor, size_t size) -> ErrorCode { @@ -181,6 +236,9 @@ ErrorCode ArchiveReaderAdaptor::try_read_archive_metadata(ZstdDecompressor& deco case ArchiveMetadataPacketType::ArchiveInfo: rc = try_read_archive_info(decompressor, packet_size); break; + case ArchiveMetadataPacketType::RangeIndex: + rc = try_read_range_index(decompressor, packet_size); + break; default: rc = try_read_unknown_metadata_packet(decompressor, packet_size); break; diff --git a/components/core/src/clp_s/ArchiveReaderAdaptor.hpp b/components/core/src/clp_s/ArchiveReaderAdaptor.hpp index 225685dac1..e03abc7805 100644 --- a/components/core/src/clp_s/ArchiveReaderAdaptor.hpp +++ b/components/core/src/clp_s/ArchiveReaderAdaptor.hpp @@ -1,10 +1,20 @@ #ifndef CLP_S_ARCHIVEREADERADAPTOR_HPP #define CLP_S_ARCHIVEREADERADAPTOR_HPP +#include #include #include #include #include +#include +#include + +// We use NOLINTNEXTLINE to satisfy clang-tidy here because while we don't use any symbols from +// `nlohmann/json.hpp` directly this code does not compile without the definition of +// `nlohmann::basic_json<>` found in the `nlohmann/json.hpp` header. +// NOLINTNEXTLINE(misc-include-cleaner) +#include +#include #include "../clp/BoundedReader.hpp" #include "../clp/ReaderInterface.hpp" @@ -15,6 +25,21 @@ #include "ZstdDecompressor.hpp" namespace clp_s { +/** + * RangeIndexEntry is a struct representing a single entry in the archive range index. + */ +struct RangeIndexEntry { + explicit RangeIndexEntry(size_t start_index, size_t end_index, nlohmann::json&& fields) + : start_index{start_index}, + end_index{end_index}, + // Note: brace initializer would make nlohmann wrap the fields object in an array. + fields(std::move(fields)) {} + + size_t start_index; + size_t end_index; + nlohmann::json fields; +}; + /** * ArchiveReaderAdaptor is an adaptor class which helps with reading single and multi-file archives * which exist on either S3 or a locally mounted file system. @@ -62,6 +87,8 @@ class ArchiveReaderAdaptor { ArchiveHeader const& get_header() const { return m_archive_header; } + std::vector const& get_range_index() const { return m_range_index; } + private: /** * Tries to read an ArchiveFileInfo packet from the archive metadata. @@ -90,6 +117,14 @@ class ArchiveReaderAdaptor { */ ErrorCode try_read_archive_info(ZstdDecompressor& decompressor, size_t size); + /** + * Tries to read a RangeIndex packet from the archive metadata. + * @param decompressor + * @param size The number of decompressed bytes making up the packet. + * @return ErrorCodeSuccess on success or the relevant ErrorCode on failure. + */ + auto try_read_range_index(ZstdDecompressor& decompressor, size_t size) -> ErrorCode; + /** * Tries to read an unknown metadata packet from the archive metadata. * @param decompressor @@ -140,6 +175,7 @@ class ArchiveReaderAdaptor { std::optional m_current_reader_holder; std::shared_ptr m_timestamp_dictionary; std::shared_ptr m_reader; + std::vector m_range_index; }; } // namespace clp_s #endif // CLP_S_ARCHIVEREADERADAPTOR_HPP diff --git a/components/core/src/clp_s/indexer/CMakeLists.txt b/components/core/src/clp_s/indexer/CMakeLists.txt index 98fefeaff7..5966764f27 100644 --- a/components/core/src/clp_s/indexer/CMakeLists.txt +++ b/components/core/src/clp_s/indexer/CMakeLists.txt @@ -90,6 +90,7 @@ target_link_libraries(indexer clp::string_utils date::date MariaDBClient::MariaDBClient + nlohmann_json::nlohmann_json OpenSSL::Crypto simdjson::simdjson spdlog::spdlog diff --git a/components/core/tests/clp_s_test_utils.cpp b/components/core/tests/clp_s_test_utils.cpp new file mode 100644 index 0000000000..243c5e55ce --- /dev/null +++ b/components/core/tests/clp_s_test_utils.cpp @@ -0,0 +1,54 @@ +#include "clp_s_test_utils.hpp" + +#include +#include + +#include + +#include "../src/clp_s/CommandLineArguments.hpp" +#include "../src/clp_s/InputConfig.hpp" +#include "../src/clp_s/JsonParser.hpp" + +void compress_archive( + std::string const& file_path, + std::string const& archive_directory, + bool single_file_archive, + bool structurize_arrays, + clp_s::CommandLineArguments::FileType file_type +) { + constexpr auto cDefaultTargetEncodedSize{8ULL * 1024 * 1024 * 1024}; // 8 GiB + constexpr auto cDefaultMaxDocumentSize{512ULL * 1024 * 1024}; // 512 MiB + constexpr auto cDefaultMinTableSize{1ULL * 1024 * 1024}; // 1 MiB + constexpr auto cDefaultCompressionLevel{3}; + constexpr auto cDefaultPrintArchiveStats{false}; + + std::filesystem::create_directory(archive_directory); + REQUIRE((std::filesystem::is_directory(archive_directory))); + + clp_s::JsonParserOption parser_option{}; + parser_option.input_paths.emplace_back( + clp_s::Path{.source = clp_s::InputSource::Filesystem, .path = file_path} + ); + parser_option.archives_dir = archive_directory; + parser_option.target_encoded_size = cDefaultTargetEncodedSize; + parser_option.max_document_size = cDefaultMaxDocumentSize; + parser_option.min_table_size = cDefaultMinTableSize; + parser_option.compression_level = cDefaultCompressionLevel; + parser_option.print_archive_stats = cDefaultPrintArchiveStats; + parser_option.structurize_arrays = structurize_arrays; + parser_option.single_file_archive = single_file_archive; + parser_option.input_file_type = file_type; + + clp_s::JsonParser parser{parser_option}; + if (clp_s::CommandLineArguments::FileType::Json == file_type) { + REQUIRE(parser.parse()); + } else if (clp_s::CommandLineArguments::FileType::KeyValueIr == file_type) { + REQUIRE(parser.parse_from_ir()); + } else { + // This branch should be unreachable. + REQUIRE(false); + } + REQUIRE_NOTHROW(parser.store()); + + REQUIRE((false == std::filesystem::is_empty(archive_directory))); +} diff --git a/components/core/tests/clp_s_test_utils.hpp b/components/core/tests/clp_s_test_utils.hpp new file mode 100644 index 0000000000..f121c76548 --- /dev/null +++ b/components/core/tests/clp_s_test_utils.hpp @@ -0,0 +1,25 @@ +#ifndef CLP_S_TEST_UTILS_HPP +#define CLP_S_TEST_UTILS_HPP +#include + +#include "../src/clp_s/CommandLineArguments.hpp" + +/** + * Compresses a file into an archive directory according to a given set of configuration options. + * + * This helper uses `REQUIRE...` statements to assert that compression was successful. + * + * @param file_path + * @param archive_directory + * @param single_file_archive + * @param structurize_arrays + * @param file_type + */ +void compress_archive( + std::string const& file_path, + std::string const& archive_directory, + bool single_file_archive, + bool structurize_arrays, + clp_s::CommandLineArguments::FileType file_type +); +#endif // CLP_S_TEST_UTILS_HPP diff --git a/components/core/tests/test-clp_s-end_to_end.cpp b/components/core/tests/test-clp_s-end_to_end.cpp index bd533159d2..5947b305e1 100644 --- a/components/core/tests/test-clp_s-end_to_end.cpp +++ b/components/core/tests/test-clp_s-end_to_end.cpp @@ -8,9 +8,10 @@ #include #include +#include "../src/clp_s/CommandLineArguments.hpp" #include "../src/clp_s/InputConfig.hpp" #include "../src/clp_s/JsonConstructor.hpp" -#include "../src/clp_s/JsonParser.hpp" +#include "clp_s_test_utils.hpp" #include "TestOutputCleaner.hpp" constexpr std::string_view cTestEndToEndArchiveDirectory{"test-end-to-end-archive"}; @@ -22,7 +23,6 @@ constexpr std::string_view cTestEndToEndInputFile{"test_no_floats_sorted.jsonl"} namespace { auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path; auto get_test_input_local_path() -> std::string; -void compress(bool structurize_arrays, bool single_file_archive); auto extract() -> std::filesystem::path; void compare(std::filesystem::path const& extracted_json_path); @@ -36,39 +36,6 @@ auto get_test_input_local_path() -> std::string { return (tests_dir / get_test_input_path_relative_to_tests_dir()).string(); } -void compress(bool structurize_arrays, bool single_file_archive) { - constexpr auto cDefaultTargetEncodedSize = 8ULL * 1024 * 1024 * 1024; // 8 GiB - constexpr auto cDefaultMaxDocumentSize = 512ULL * 1024 * 1024; // 512 MiB - constexpr auto cDefaultMinTableSize = 1ULL * 1024 * 1024; // 1 MiB - constexpr auto cDefaultCompressionLevel = 3; - constexpr auto cDefaultPrintArchiveStats = false; - - std::filesystem::create_directory(cTestEndToEndArchiveDirectory); - REQUIRE((std::filesystem::is_directory(cTestEndToEndArchiveDirectory))); - - clp_s::JsonParserOption parser_option{}; - parser_option.input_paths.emplace_back( - clp_s::Path{ - .source = clp_s::InputSource::Filesystem, - .path = get_test_input_local_path() - } - ); - parser_option.archives_dir = cTestEndToEndArchiveDirectory; - parser_option.target_encoded_size = cDefaultTargetEncodedSize; - parser_option.max_document_size = cDefaultMaxDocumentSize; - parser_option.min_table_size = cDefaultMinTableSize; - parser_option.compression_level = cDefaultCompressionLevel; - parser_option.print_archive_stats = cDefaultPrintArchiveStats; - parser_option.structurize_arrays = structurize_arrays; - parser_option.single_file_archive = single_file_archive; - - clp_s::JsonParser parser{parser_option}; - REQUIRE(parser.parse()); - parser.store(); - - REQUIRE((false == std::filesystem::is_empty(cTestEndToEndArchiveDirectory))); -} - auto extract() -> std::filesystem::path { constexpr auto cDefaultOrdered = false; constexpr auto cDefaultTargetOrderedChunkSize = 0; @@ -135,7 +102,13 @@ TEST_CASE("clp-s-compress-extract-no-floats", "[clp-s][end-to-end]") { std::string{cTestEndToEndOutputSortedJson}} }; - compress(structurize_arrays, single_file_archive); + REQUIRE_NOTHROW(compress_archive( + get_test_input_local_path(), + std::string{cTestEndToEndArchiveDirectory}, + single_file_archive, + structurize_arrays, + clp_s::CommandLineArguments::FileType::Json + )); auto extracted_json_path = extract(); diff --git a/components/core/tests/test-clp_s-range_index.cpp b/components/core/tests/test-clp_s-range_index.cpp new file mode 100644 index 0000000000..447a189c2b --- /dev/null +++ b/components/core/tests/test-clp_s-range_index.cpp @@ -0,0 +1,189 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "../src/clp/ffi/ir_stream/protocol_constants.hpp" +#include "../src/clp/ffi/ir_stream/Serializer.hpp" +#include "../src/clp/FileReader.hpp" +#include "../src/clp/FileWriter.hpp" +#include "../src/clp/ir/types.hpp" +#include "../src/clp/streaming_compression/zstd/Compressor.hpp" +#include "../src/clp/type_utils.hpp" +#include "../src/clp_s/archive_constants.hpp" +#include "../src/clp_s/ArchiveReader.hpp" +#include "../src/clp_s/InputConfig.hpp" +#include "clp_s_test_utils.hpp" +#include "TestOutputCleaner.hpp" + +constexpr std::string_view cTestRangeIndexArchiveDirectory{"test-range-index-archive"}; +constexpr std::string_view cTestRangeIndexIRDirectory{"test-range-index-ir"}; +constexpr std::string_view cTestRangeIndexInputFileDirectory{"test_log_files"}; +constexpr std::string_view cTestRangeIndexInputFile{"test_no_floats_sorted.jsonl"}; +constexpr std::string_view cTestRangeIndexIRInputFile{"test_no_floats_sorted.ir"}; +constexpr std::string_view cTestRangeIndexIRMetadataKey{"test_key"}; +constexpr std::string_view cTestRangeIndexIRMetadataValue{"test_value"}; + +namespace { +auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path; +auto get_test_input_local_path() -> std::string; +void serialize_record( + nlohmann::json const& auto_gen, + nlohmann::json const& user_gen, + clp::ffi::ir_stream::Serializer& serializer +); +void generate_ir(); +void check_archive_metadata(bool from_ir); + +auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path { + return std::filesystem::path{cTestRangeIndexInputFileDirectory} / cTestRangeIndexInputFile; +} + +auto get_test_input_local_path() -> std::string { + std::filesystem::path const current_file_path{__FILE__}; + auto const tests_dir{current_file_path.parent_path()}; + return (tests_dir / get_test_input_path_relative_to_tests_dir()).string(); +} + +auto get_ir_test_input_relative_path() -> std::string { + return (std::filesystem::path{cTestRangeIndexIRDirectory} / cTestRangeIndexIRInputFile) + .string(); +} + +void serialize_record( + nlohmann::json const& auto_gen, + nlohmann::json const& user_gen, + clp::ffi::ir_stream::Serializer& serializer +) { + auto const auto_gen_bytes{nlohmann::json::to_msgpack(auto_gen)}; + auto const user_gen_bytes{nlohmann::json::to_msgpack(user_gen)}; + auto const auto_gen_handle{msgpack::unpack( + clp::size_checked_pointer_cast(auto_gen_bytes.data()), + auto_gen_bytes.size() + )}; + auto const user_gen_handle{msgpack::unpack( + clp::size_checked_pointer_cast(user_gen_bytes.data()), + user_gen_bytes.size() + )}; + auto const auto_gen_obj{auto_gen_handle.get()}; + auto const user_gen_obj{user_gen_handle.get()}; + REQUIRE(msgpack::type::MAP == auto_gen_obj.type); + REQUIRE(msgpack::type::MAP == user_gen_obj.type); + REQUIRE(serializer.serialize_msgpack_map(auto_gen_obj.via.map, user_gen_obj.via.map)); +} + +void generate_ir() { + std::filesystem::create_directory(cTestRangeIndexIRDirectory); + REQUIRE(std::filesystem::is_directory(cTestRangeIndexIRDirectory)); + + nlohmann::json ir_metadata = {{cTestRangeIndexIRMetadataKey, cTestRangeIndexIRMetadataValue}}; + + auto result{clp::ffi::ir_stream::Serializer::create( + ir_metadata + )}; + REQUIRE(false == result.has_error()); + auto& serializer = result.value(); + + auto empty_object = nlohmann::json::parse("{}"); + clp::FileReader reader(get_test_input_local_path()); + std::string line; + while (clp::ErrorCode::ErrorCode_Success + == reader.try_read_to_delimiter('\n', false, false, line)) + { + auto json_line = nlohmann::json::parse(line); + serialize_record(empty_object, json_line, serializer); + } + + clp::FileWriter writer; + REQUIRE_NOTHROW(writer.open( + get_ir_test_input_relative_path(), + clp::FileWriter::OpenMode::CREATE_FOR_WRITING + )); + clp::streaming_compression::zstd::Compressor compressor; + compressor.open(writer); + + auto const eof_packet{clp::ffi::ir_stream::cProtocol::Eof}; + auto const ir_buf{serializer.get_ir_buf_view()}; + compressor.write(clp::size_checked_pointer_cast(ir_buf.data()), ir_buf.size()); + compressor.write(clp::size_checked_pointer_cast(&eof_packet), sizeof(eof_packet)); + compressor.close(); + writer.close(); +} + +void check_archive_metadata(bool from_ir) { + clp_s::ArchiveReader archive_reader; + auto const expected_input_path{ + from_ir ? get_ir_test_input_relative_path() : get_test_input_local_path() + }; + for (auto const& entry : std::filesystem::directory_iterator(cTestRangeIndexArchiveDirectory)) { + clp_s::Path archive_path{ + .source{clp_s::InputSource::Filesystem}, + .path{entry.path().string()} + }; + REQUIRE_NOTHROW(archive_reader.open(archive_path, clp_s::NetworkAuthOption{})); + auto const& range_index = archive_reader.get_range_index(); + REQUIRE(1ULL == range_index.size()); + auto const& range_index_entry = range_index.front(); + REQUIRE(0ULL == range_index_entry.start_index); + REQUIRE(4ULL == range_index_entry.end_index); + auto const& metadata_fields = range_index_entry.fields; + REQUIRE(metadata_fields.contains(clp_s::constants::range_index::cArchiveCreatorId)); + REQUIRE(metadata_fields.at(clp_s::constants::range_index::cArchiveCreatorId).is_string()); + REQUIRE(false + == metadata_fields.at(clp_s::constants::range_index::cArchiveCreatorId) + .template get() + .empty()); + REQUIRE(metadata_fields.contains(clp_s::constants::range_index::cFilename)); + REQUIRE(metadata_fields.at(clp_s::constants::range_index::cFilename).is_string()); + REQUIRE(expected_input_path + == metadata_fields.at(clp_s::constants::range_index::cFilename) + .template get()); + REQUIRE(metadata_fields.contains(clp_s::constants::range_index::cFileSplitNumber)); + REQUIRE(metadata_fields.at(clp_s::constants::range_index::cFileSplitNumber) + .is_number_integer()); + REQUIRE(0ULL + == metadata_fields.at(clp_s::constants::range_index::cFileSplitNumber) + .template get()); + if (from_ir) { + REQUIRE(metadata_fields.contains(cTestRangeIndexIRMetadataKey)); + REQUIRE(metadata_fields.at(cTestRangeIndexIRMetadataKey).is_string()); + REQUIRE( + cTestRangeIndexIRMetadataValue + == metadata_fields.at(cTestRangeIndexIRMetadataKey).template get() + ); + } + REQUIRE_NOTHROW(archive_reader.close()); + } +} +} // namespace + +TEST_CASE("clp-s-range-index", "[clp-s][range-index]") { + auto single_file_archive = GENERATE(true, false); + auto from_ir = GENERATE(true, false); + + TestOutputCleaner const test_cleanup{ + {std::string{cTestRangeIndexArchiveDirectory}, std::string{cTestRangeIndexIRDirectory}} + }; + + auto input_file{get_test_input_local_path()}; + auto input_file_type{clp_s::CommandLineArguments::FileType::Json}; + if (from_ir) { + generate_ir(); + input_file = get_ir_test_input_relative_path(); + input_file_type = clp_s::CommandLineArguments::FileType::KeyValueIr; + } + REQUIRE_NOTHROW(compress_archive( + input_file, + std::string{cTestRangeIndexArchiveDirectory}, + single_file_archive, + false, + input_file_type + )); + check_archive_metadata(from_ir); +} diff --git a/components/core/tests/test-clp_s-search.cpp b/components/core/tests/test-clp_s-search.cpp index 54d64e19d2..e92501c219 100644 --- a/components/core/tests/test-clp_s-search.cpp +++ b/components/core/tests/test-clp_s-search.cpp @@ -14,8 +14,8 @@ #include #include "../src/clp_s/ArchiveReader.hpp" +#include "../src/clp_s/CommandLineArguments.hpp" #include "../src/clp_s/InputConfig.hpp" -#include "../src/clp_s/JsonParser.hpp" #include "../src/clp_s/search/ast/ConvertToExists.hpp" #include "../src/clp_s/search/ast/EmptyExpr.hpp" #include "../src/clp_s/search/ast/Expression.hpp" @@ -28,6 +28,7 @@ #include "../src/clp_s/search/Projection.hpp" #include "../src/clp_s/search/SchemaMatch.hpp" #include "../src/clp_s/Utils.hpp" +#include "clp_s_test_utils.hpp" #include "TestOutputCleaner.hpp" constexpr std::string_view cTestSearchArchiveDirectory{"test-clp-s-search-archive"}; @@ -38,7 +39,6 @@ constexpr std::string_view cTestIdxKey{"idx"}; namespace { auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path; auto get_test_input_local_path() -> std::string; -void compress(bool structurize_arrays, bool single_file_archive); void search(std::string const& query, bool ignore_case, std::vector const& expected_results); void validate_results( @@ -56,39 +56,6 @@ auto get_test_input_local_path() -> std::string { return (tests_dir / get_test_input_path_relative_to_tests_dir()).string(); } -void compress(bool structurize_arrays, bool single_file_archive) { - constexpr auto cDefaultTargetEncodedSize = 8ULL * 1024 * 1024 * 1024; // 8 GiB - constexpr auto cDefaultMaxDocumentSize = 512ULL * 1024 * 1024; // 512 MiB - constexpr auto cDefaultMinTableSize = 1ULL * 1024 * 1024; // 1 MiB - constexpr auto cDefaultCompressionLevel = 3; - constexpr auto cDefaultPrintArchiveStats = false; - - std::filesystem::create_directory(cTestSearchArchiveDirectory); - REQUIRE((std::filesystem::is_directory(cTestSearchArchiveDirectory))); - - clp_s::JsonParserOption parser_option{}; - parser_option.input_paths.emplace_back( - clp_s::Path{ - .source = clp_s::InputSource::Filesystem, - .path = get_test_input_local_path() - } - ); - parser_option.archives_dir = cTestSearchArchiveDirectory; - parser_option.target_encoded_size = cDefaultTargetEncodedSize; - parser_option.max_document_size = cDefaultMaxDocumentSize; - parser_option.min_table_size = cDefaultMinTableSize; - parser_option.compression_level = cDefaultCompressionLevel; - parser_option.print_archive_stats = cDefaultPrintArchiveStats; - parser_option.structurize_arrays = structurize_arrays; - parser_option.single_file_archive = single_file_archive; - - clp_s::JsonParser parser{parser_option}; - REQUIRE(parser.parse()); - parser.store(); - - REQUIRE((false == std::filesystem::is_empty(cTestSearchArchiveDirectory))); -} - void validate_results( std::vector const& results, std::vector const& expected_results @@ -188,7 +155,13 @@ TEST_CASE("clp-s-search", "[clp-s][search]") { TestOutputCleaner const test_cleanup{{std::string{cTestSearchArchiveDirectory}}}; - REQUIRE_NOTHROW(compress(structurize_arrays, single_file_archive)); + REQUIRE_NOTHROW(compress_archive( + get_test_input_local_path(), + std::string{cTestSearchArchiveDirectory}, + single_file_archive, + structurize_arrays, + clp_s::CommandLineArguments::FileType::Json + )); for (auto const& [query, expected_results] : queries_and_results) { REQUIRE_NOTHROW(search(query, false, expected_results));