From e68024d1ff372340e4879bd2d35f629b273b43d3 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Tue, 25 Jun 2024 15:02:26 +0000 Subject: [PATCH 1/3] Fix bug where print-archive-stats option not initialized --- components/core/src/clp_s/JsonParser.cpp | 9 ++++++--- components/core/src/clp_s/JsonParser.hpp | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/components/core/src/clp_s/JsonParser.cpp b/components/core/src/clp_s/JsonParser.cpp index c4fe7a43e..dff676e85 100644 --- a/components/core/src/clp_s/JsonParser.cpp +++ b/components/core/src/clp_s/JsonParser.cpp @@ -17,7 +17,8 @@ JsonParser::JsonParser(JsonParserOption const& option) m_target_encoded_size(option.target_encoded_size), m_max_document_size(option.max_document_size), m_timestamp_key(option.timestamp_key), - m_structurize_arrays(option.structurize_arrays) { + m_structurize_arrays(option.structurize_arrays), + m_print_archive_stats(option.print_archive_stats) { if (false == FileUtils::validate_path(option.file_paths)) { exit(1); } @@ -30,11 +31,12 @@ JsonParser::JsonParser(JsonParserOption const& option) FileUtils::find_all_files(file_path, m_file_paths); } - ArchiveWriterOption archive_writer_option; + ArchiveWriterOption archive_writer_option{}; archive_writer_option.archives_dir = m_archives_dir; archive_writer_option.id = m_generator(); archive_writer_option.compression_level = option.compression_level; archive_writer_option.print_archive_stats = option.print_archive_stats; + archive_writer_option.print_archive_stats = m_print_archive_stats; m_archive_writer = std::make_unique(option.metadata_db); m_archive_writer->open(archive_writer_option); @@ -506,10 +508,11 @@ void JsonParser::store() { void JsonParser::split_archive() { m_archive_writer->close(); - ArchiveWriterOption archive_writer_option; + ArchiveWriterOption archive_writer_option{}; archive_writer_option.archives_dir = m_archives_dir; archive_writer_option.id = m_generator(); archive_writer_option.compression_level = m_compression_level; + archive_writer_option.print_archive_stats = m_print_archive_stats; m_archive_writer->open(archive_writer_option); } diff --git a/components/core/src/clp_s/JsonParser.hpp b/components/core/src/clp_s/JsonParser.hpp index a0010e66f..c191d9807 100644 --- a/components/core/src/clp_s/JsonParser.hpp +++ b/components/core/src/clp_s/JsonParser.hpp @@ -109,6 +109,7 @@ class JsonParser { size_t m_target_encoded_size; size_t m_max_document_size; bool m_structurize_arrays{false}; + bool m_print_archive_stats{false}; }; } // namespace clp_s From 242cf5f7b139aa05098fb990a0577bbbff156248 Mon Sep 17 00:00:00 2001 From: Devin Gibson Date: Tue, 25 Jun 2024 17:07:15 -0400 Subject: [PATCH 2/3] Update components/core/src/clp_s/JsonParser.cpp Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/src/clp_s/JsonParser.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/core/src/clp_s/JsonParser.cpp b/components/core/src/clp_s/JsonParser.cpp index dff676e85..b64b70f07 100644 --- a/components/core/src/clp_s/JsonParser.cpp +++ b/components/core/src/clp_s/JsonParser.cpp @@ -35,7 +35,6 @@ JsonParser::JsonParser(JsonParserOption const& option) archive_writer_option.archives_dir = m_archives_dir; archive_writer_option.id = m_generator(); archive_writer_option.compression_level = option.compression_level; - archive_writer_option.print_archive_stats = option.print_archive_stats; archive_writer_option.print_archive_stats = m_print_archive_stats; m_archive_writer = std::make_unique(option.metadata_db); From aae53dd444e81d81d6aff01e09b470e2e7d00877 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Tue, 25 Jun 2024 21:29:36 +0000 Subject: [PATCH 3/3] Store ArchiveWriterOption as a member of JsonParser --- components/core/src/clp_s/JsonParser.cpp | 28 ++++++++---------------- components/core/src/clp_s/JsonParser.hpp | 5 +---- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/components/core/src/clp_s/JsonParser.cpp b/components/core/src/clp_s/JsonParser.cpp index b64b70f07..d73643a64 100644 --- a/components/core/src/clp_s/JsonParser.cpp +++ b/components/core/src/clp_s/JsonParser.cpp @@ -11,14 +11,11 @@ namespace clp_s { JsonParser::JsonParser(JsonParserOption const& option) - : m_archives_dir(option.archives_dir), - m_num_messages(0), - m_compression_level(option.compression_level), + : m_num_messages(0), m_target_encoded_size(option.target_encoded_size), m_max_document_size(option.max_document_size), m_timestamp_key(option.timestamp_key), - m_structurize_arrays(option.structurize_arrays), - m_print_archive_stats(option.print_archive_stats) { + m_structurize_arrays(option.structurize_arrays) { if (false == FileUtils::validate_path(option.file_paths)) { exit(1); } @@ -31,14 +28,13 @@ JsonParser::JsonParser(JsonParserOption const& option) FileUtils::find_all_files(file_path, m_file_paths); } - ArchiveWriterOption archive_writer_option{}; - archive_writer_option.archives_dir = m_archives_dir; - archive_writer_option.id = m_generator(); - archive_writer_option.compression_level = option.compression_level; - archive_writer_option.print_archive_stats = m_print_archive_stats; + m_archive_options.archives_dir = option.archives_dir; + m_archive_options.compression_level = option.compression_level; + m_archive_options.print_archive_stats = option.print_archive_stats; + m_archive_options.id = m_generator(); m_archive_writer = std::make_unique(option.metadata_db); - m_archive_writer->open(archive_writer_option); + m_archive_writer->open(m_archive_options); } void JsonParser::parse_obj_in_array(ondemand::object line, int32_t parent_node_id) { @@ -506,14 +502,8 @@ void JsonParser::store() { void JsonParser::split_archive() { m_archive_writer->close(); - - ArchiveWriterOption archive_writer_option{}; - archive_writer_option.archives_dir = m_archives_dir; - archive_writer_option.id = m_generator(); - archive_writer_option.compression_level = m_compression_level; - archive_writer_option.print_archive_stats = m_print_archive_stats; - - m_archive_writer->open(archive_writer_option); + m_archive_options.id = m_generator(); + m_archive_writer->open(m_archive_options); } } // namespace clp_s diff --git a/components/core/src/clp_s/JsonParser.hpp b/components/core/src/clp_s/JsonParser.hpp index c191d9807..f65f4c4b1 100644 --- a/components/core/src/clp_s/JsonParser.hpp +++ b/components/core/src/clp_s/JsonParser.hpp @@ -93,10 +93,7 @@ class JsonParser { void split_archive(); int m_num_messages; - int m_compression_level; std::vector m_file_paths; - std::string m_archives_dir; - std::string m_schema_tree_path; Schema m_current_schema; ParsedMessage m_current_parsed_message; @@ -106,10 +103,10 @@ class JsonParser { boost::uuids::random_generator m_generator; std::unique_ptr m_archive_writer; + ArchiveWriterOption m_archive_options{}; size_t m_target_encoded_size; size_t m_max_document_size; bool m_structurize_arrays{false}; - bool m_print_archive_stats{false}; }; } // namespace clp_s