refactor(clp-package): Unify the metadata schema for JSON and IR stre…

…ams. (#620)
y-scope · Dec 6, 2024 · 60d85d0 · 60d85d0
1 parent 5639dfc
commit 60d85d0
Show file tree

Hide file tree

Showing 9 changed files with 19 additions and 35 deletions.
diff --git a/components/core/src/clp/clo/OutputHandler.cpp b/components/core/src/clp/clo/OutputHandler.cpp
@@ -100,7 +100,7 @@ ErrorCode ResultsCacheOutputHandler::flush() {
         try {
             m_results.emplace_back(std::move(bsoncxx::builder::basic::make_document(
                     bsoncxx::builder::basic::kvp(
-                            cResultsCacheKeys::OrigFileId,
+                            cResultsCacheKeys::SearchOutput::OrigFileId,
                             std::move(result.orig_file_id)
                     ),
                     bsoncxx::builder::basic::kvp(

diff --git a/components/core/src/clp/clo/clo.cpp b/components/core/src/clp/clo/clo.cpp
@@ -171,7 +171,7 @@ bool extract_ir(CommandLineArguments const& command_line_args) {
                                      string const& orig_file_id,
                                      size_t begin_message_ix,
                                      size_t end_message_ix,
-                                     bool is_last_ir_chunk) {
+                                     bool is_last_chunk) {
             auto dest_ir_file_name = orig_file_id;
             dest_ir_file_name += "_" + std::to_string(begin_message_ix);
             dest_ir_file_name += "_" + std::to_string(end_message_ix);
@@ -195,13 +195,9 @@ bool extract_ir(CommandLineArguments const& command_line_args) {
                             dest_ir_file_name
                     ),
                     bsoncxx::builder::basic::kvp(
-                            clp::clo::cResultsCacheKeys::OrigFileId,
+                            clp::clo::cResultsCacheKeys::IrOutput::StreamId,
                             orig_file_id
                     ),
-                    bsoncxx::builder::basic::kvp(
-                            clp::clo::cResultsCacheKeys::IrOutput::FileSplitId,
-                            file_split_id
-                    ),
                     bsoncxx::builder::basic::kvp(
                             clp::clo::cResultsCacheKeys::IrOutput::BeginMsgIx,
                             static_cast<int64_t>(begin_message_ix)
@@ -211,8 +207,8 @@ bool extract_ir(CommandLineArguments const& command_line_args) {
                             static_cast<int64_t>(end_message_ix)
                     ),
                     bsoncxx::builder::basic::kvp(
-                            clp::clo::cResultsCacheKeys::IrOutput::IsLastIrChunk,
-                            is_last_ir_chunk
+                            clp::clo::cResultsCacheKeys::IrOutput::IsLastChunk,
+                            is_last_chunk
                     )
             )));
             return true;

diff --git a/components/core/src/clp/clo/constants.hpp b/components/core/src/clp/clo/constants.hpp
@@ -3,17 +3,16 @@
 
 // NOLINTBEGIN(cppcoreguidelines-avoid-c-arrays, readability-identifier-naming)
 namespace clp::clo::cResultsCacheKeys {
-constexpr char OrigFileId[]{"orig_file_id"};
-
 namespace IrOutput {
 constexpr char Path[]{"path"};
-constexpr char FileSplitId[]{"file_split_id"};
+constexpr char StreamId[]{"stream_id"};
 constexpr char BeginMsgIx[]{"begin_msg_ix"};
 constexpr char EndMsgIx[]{"end_msg_ix"};
-constexpr char IsLastIrChunk[]{"is_last_ir_chunk"};
+constexpr char IsLastChunk[]{"is_last_chunk"};
 }  // namespace IrOutput
 
 namespace SearchOutput {
+constexpr char OrigFileId[]{"orig_file_id"};
 constexpr char OrigFilePath[]{"orig_file_path"};
 constexpr char LogEventIx[]{"log_event_ix"};
 constexpr char Timestamp[]{"timestamp"};

diff --git a/components/core/src/clp/clp/FileDecompressor.hpp b/components/core/src/clp/clp/FileDecompressor.hpp
@@ -39,7 +39,7 @@ class FileDecompressor {
      *
      * @tparam IrOutputHandler Function to handle the resulting IR chunks.
      * Signature: (std::filesystem::path const& ir_file_path, string const& orig_file_id,
-     * size_t begin_message_ix, size_t end_message_ix, bool is_last_ir_chunk) -> bool;
+     * size_t begin_message_ix, size_t end_message_ix, bool is_last_chunk) -> bool;
      * The function returns whether it succeeded.
      * @param archive_reader
      * @param file_metadata_ix

diff --git a/components/core/src/clp/clp/decompression.cpp b/components/core/src/clp/clp/decompression.cpp
@@ -282,7 +282,7 @@ bool decompress_to_ir(CommandLineArguments& command_line_args) {
                                      string const& orig_file_id,
                                      size_t begin_message_ix,
                                      size_t end_message_ix,
-                                     [[maybe_unused]] bool is_last_ir_chunk) {
+                                     [[maybe_unused]] bool is_last_chunk) {
             auto dest_ir_file_name = orig_file_id;
             dest_ir_file_name += "_" + std::to_string(begin_message_ix);
             dest_ir_file_name += "_" + std::to_string(end_message_ix);

diff --git a/components/core/src/clp_s/JsonConstructor.cpp b/components/core/src/clp_s/JsonConstructor.cpp
@@ -122,7 +122,7 @@ void JsonConstructor::construct_in_order() {
                             new_file_path.filename()
                     ),
                     bsoncxx::builder::basic::kvp(
-                            constants::results_cache::decompression::cOrigFileId,
+                            constants::results_cache::decompression::cStreamId,
                             m_option.archive_id
                     ),
                     bsoncxx::builder::basic::kvp(
@@ -134,7 +134,7 @@ void JsonConstructor::construct_in_order() {
                             last_idx
                     ),
                     bsoncxx::builder::basic::kvp(
-                            constants::results_cache::decompression::cIsLastIrChunk,
+                            constants::results_cache::decompression::cIsLastChunk,
                             false == open_new_writer
                     )
             )));

diff --git a/components/core/src/clp_s/archive_constants.hpp b/components/core/src/clp_s/archive_constants.hpp
@@ -29,10 +29,10 @@ constexpr char cLogEventIdxName[] = "log_event_idx";
 
 namespace results_cache::decompression {
 constexpr char cPath[]{"path"};
-constexpr char cOrigFileId[]{"orig_file_id"};
+constexpr char cStreamId[]{"stream_id"};
 constexpr char cBeginMsgIx[]{"begin_msg_ix"};
 constexpr char cEndMsgIx[]{"end_msg_ix"};
-constexpr char cIsLastIrChunk[]{"is_last_ir_chunk"};
+constexpr char cIsLastChunk[]{"is_last_chunk"};
 }  // namespace results_cache::decompression
 
 namespace results_cache::search {

diff --git a/components/log-viewer-webui/client/src/api/query.js b/components/log-viewer-webui/client/src/api/query.js
@@ -2,22 +2,11 @@ import axios from "axios";
 
 
 /**
- * @typedef {object} ExtractIrResp
+ * @typedef {object} ExtractStreamResp
+ * @property {string} stream_id
  * @property {number} begin_msg_ix
  * @property {number} end_msg_ix
- * @property {string} file_split_id
- * @property {boolean} is_last_ir_chunk
- * @property {string} orig_file_id
- * @property {string} path
- * @property {string} _id
- */
-
-/**
- * @typedef {object} ExtractJsonResp
- * @property {number} begin_msg_ix
- * @property {number} end_msg_ix
- * @property {boolean} is_last_ir_chunk
- * @property {string} orig_file_id
+ * @property {boolean} is_last_chunk
  * @property {string} path
  * @property {string} _id
  */
@@ -30,7 +19,7 @@ import axios from "axios";
  * @param {string} streamId
  * @param {number} logEventIdx
  * @param {Function} onUploadProgress Callback to handle upload progress events.
- * @return {Promise<axios.AxiosResponse<ExtractIrResp|ExtractJsonResp>>}
+ * @return {Promise<axios.AxiosResponse<ExtractStreamResp>>}
  */
 const submitExtractStreamJob = async (extractJobType, streamId, logEventIdx, onUploadProgress) => {
     return await axios.post(

diff --git a/components/log-viewer-webui/server/src/DbManager.js b/components/log-viewer-webui/server/src/DbManager.js
@@ -171,7 +171,7 @@ class DbManager {
      */
     async getExtractedStreamFileMetadata (streamId, logEventIdx) {
         return await this.#streamFilesCollection.findOne({
-            orig_file_id: streamId,
+            stream_id: streamId,
             begin_msg_ix: {$lte: logEventIdx},
             end_msg_ix: {$gt: logEventIdx},
         });