Prepare for upcoming changes from CLP-s

y-scope · Nov 19, 2024 · ce1acdb · ce1acdb
1 parent ae6a601
commit ce1acdb
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 9 deletions.
diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py
@@ -936,6 +936,7 @@ def start_log_viewer_webui(
         "MongoDbStreamFilesCollectionName": clp_config.results_cache.stream_collection_name,
         "ClientDir": str(container_log_viewer_webui_dir / "client"),
         "StreamFilesDir": str(container_clp_config.stream_output.directory),
+        "StreamTargetUncompressedSize": container_clp_config.stream_output.target_uncompressed_size,
         "LogViewerDir": str(container_log_viewer_webui_dir / "yscope-log-viewer"),
     }
     settings_json = read_and_update_settings_json(settings_json_path, settings_json_updates)

diff --git a/components/log-viewer-webui/server/settings.json b/components/log-viewer-webui/server/settings.json
@@ -10,5 +10,6 @@
 
     "ClientDir": "../client/dist",
     "StreamFilesDir": "../../../build/clp-package/var/data/streams",
+    "StreamTargetUncompressedSize": 134217728,
     "LogViewerDir": "../yscope-log-viewer/dist"
 }
diff --git a/components/log-viewer-webui/server/src/routes/query.js b/components/log-viewer-webui/server/src/routes/query.js
@@ -1,12 +1,7 @@
+import settings from "../../settings.json" with {type: "json"};
 import {QUERY_JOB_TYPE} from "../DbManager.js";
 
 
-// eslint-disable-next-line no-magic-numbers
-const EXTRACT_IR_TARGET_UNCOMPRESSED_SIZE = 128 * 1024 * 1024;
-// eslint-disable-next-line no-magic-numbers
-const EXTRACT_JSON_TARGET_CHUNK_SIZE = 100 * 1000;
-
-
 /**
  * Submits a stream extraction job with the given parameters and waits for it.
  *
@@ -23,17 +18,18 @@ const submitAndWaitForExtractStreamJob = async (
     sanitizedLogEventIdx
 ) => {
     let jobConfig;
+    const streamTargetUncompressedSize = settings.StreamTargetUncompressedSize;
     if (QUERY_JOB_TYPE.EXTRACT_IR === extractJobType) {
         jobConfig = {
             file_split_id: null,
             msg_ix: sanitizedLogEventIdx,
             orig_file_id: streamId,
-            target_uncompressed_size: EXTRACT_IR_TARGET_UNCOMPRESSED_SIZE,
+            target_uncompressed_size: streamTargetUncompressedSize,
         };
     } else if (QUERY_JOB_TYPE.EXTRACT_JSON === extractJobType) {
         jobConfig = {
             archive_id: streamId,
-            target_chunk_size: EXTRACT_JSON_TARGET_CHUNK_SIZE,
+            target_chunk_size: streamTargetUncompressedSize,
         };
     }
 

diff --git a/components/package-template/src/etc/clp-config.yml b/components/package-template/src/etc/clp-config.yml
@@ -86,7 +86,7 @@
 #stream_output:
 #  directory: "var/data/streams"
 #
-#  # How large each IR file should be before being split into a new IR file
+#  # How large each stream file should be before being split into a new stream file
 #  target_uncompressed_size: 134217728  # 128 MB
 #
 ## Location where other data (besides archives) are stored. It will be created if