From 4931f6b17cf863e7cfdf59e38df28d48dc92fed2 Mon Sep 17 00:00:00 2001 From: davidlion Date: Thu, 28 Aug 2025 11:20:07 -0400 Subject: [PATCH 01/91] feat(log-surgeon)!: Add support for a single capture group in a schema rule to have parity with the heuristic parser. --- components/core/src/clp/Utils.cpp | 11 +++ .../clp/streaming_archive/writer/Archive.cpp | 92 ++++++++++++++----- .../core/tests/test-ParserWithUserSchema.cpp | 23 +++++ .../multiple_capture_groups.txt | 1 + .../single_capture_group.txt | 1 + 5 files changed, 105 insertions(+), 23 deletions(-) create mode 100644 components/core/tests/test_schema_files/multiple_capture_groups.txt create mode 100644 components/core/tests/test_schema_files/single_capture_group.txt diff --git a/components/core/src/clp/Utils.cpp b/components/core/src/clp/Utils.cpp index f52cb87cdf..8fcf17ff26 100644 --- a/components/core/src/clp/Utils.cpp +++ b/components/core/src/clp/Utils.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -188,6 +189,16 @@ load_lexer_from_file(std::string const& schema_file_path, log_surgeon::lexers::B for (std::unique_ptr const& parser_ast : schema_ast->m_schema_vars) { auto* rule = dynamic_cast(parser_ast.get()); + // Currently, we only support at most a single capture group in each variable. If a capture + // group is present its match will be treated as the variable rather than the full match. + if (1 < rule->m_regex_ptr->get_subtree_positive_captures().size()) { + throw std::runtime_error( + schema_file_path + ":" + std::to_string(rule->m_line_num + 1) + + ": error: the schema rule '" + rule->m_name + + "' has a regex pattern containing > 1 capture group.\n" + ); + } + if ("timestamp" == rule->m_name) { continue; } diff --git a/components/core/src/clp/streaming_archive/writer/Archive.cpp b/components/core/src/clp/streaming_archive/writer/Archive.cpp index a5fe8a29f9..b1edd0c9ce 100644 --- a/components/core/src/clp/streaming_archive/writer/Archive.cpp +++ b/components/core/src/clp/streaming_archive/writer/Archive.cpp @@ -2,14 +2,17 @@ #include +#include #include -#include -#include +#include +#include +#include #include #include #include #include +#include #include #include #include @@ -23,11 +26,7 @@ using clp::ir::eight_byte_encoded_variable_t; using clp::ir::four_byte_encoded_variable_t; -using log_surgeon::LogEventView; -using std::list; -using std::make_unique; using std::string; -using std::unordered_set; using std::vector; namespace clp::streaming_archive::writer { @@ -315,13 +314,13 @@ Archive::write_msg(epochtime_t timestamp, string const& message, size_t num_unco update_segment_indices(logtype_id, var_ids); } -void Archive::write_msg_using_schema(LogEventView const& log_view) { +void Archive::write_msg_using_schema(log_surgeon::LogEventView const& log_view) { epochtime_t timestamp = 0; TimestampPattern* timestamp_pattern = nullptr; auto const& log_output_buffer = log_view.get_log_output_buffer(); if (log_output_buffer->has_timestamp()) { - size_t start; - size_t end; + size_t start{}; + size_t end{}; timestamp_pattern = (TimestampPattern*)TimestampPattern::search_known_ts_patterns( log_output_buffer->get_mutable_token(0).to_string(), timestamp, @@ -360,7 +359,7 @@ void Archive::write_msg_using_schema(LogEventView const& log_view) { if (timestamp_pattern == nullptr) { start_pos = log_output_buffer->get_token(1).m_start_pos; } - uint32_t end_pos = log_output_buffer->get_token(log_output_buffer->pos() - 1).m_end_pos; + uint32_t const end_pos = log_output_buffer->get_token(log_output_buffer->pos() - 1).m_end_pos; if (start_pos <= end_pos) { num_uncompressed_bytes = end_pos - start_pos; } else { @@ -369,7 +368,7 @@ void Archive::write_msg_using_schema(LogEventView const& log_view) { } for (uint32_t i = 1; i < log_output_buffer->pos(); i++) { log_surgeon::Token& token = log_output_buffer->get_mutable_token(i); - int token_type = token.m_type_ids_ptr->at(0); + auto const token_type{token.m_type_ids_ptr->at(0)}; if (log_output_buffer->has_delimiters() && (timestamp_pattern != nullptr || i > 1) && token_type != static_cast(log_surgeon::SymbolId::TokenUncaughtString) && token_type != static_cast(log_surgeon::SymbolId::TokenNewline)) @@ -388,13 +387,13 @@ void Archive::write_msg_using_schema(LogEventView const& log_view) { break; } case static_cast(log_surgeon::SymbolId::TokenInt): { - encoded_variable_t encoded_var; + encoded_variable_t encoded_var{}; if (!EncodedVariableInterpreter::convert_string_to_representable_integer_var( token.to_string(), encoded_var )) { - variable_dictionary_id_t id; + variable_dictionary_id_t id{}; m_var_dict.add_entry(token.to_string(), id); encoded_var = EncodedVariableInterpreter::encode_var_dict_id(id); m_logtype_dict_entry.add_dictionary_var(); @@ -405,13 +404,13 @@ void Archive::write_msg_using_schema(LogEventView const& log_view) { break; } case static_cast(log_surgeon::SymbolId::TokenFloat): { - encoded_variable_t encoded_var; + encoded_variable_t encoded_var{}; if (!EncodedVariableInterpreter::convert_string_to_representable_float_var( token.to_string(), encoded_var )) { - variable_dictionary_id_t id; + variable_dictionary_id_t id{}; m_var_dict.add_entry(token.to_string(), id); encoded_var = EncodedVariableInterpreter::encode_var_dict_id(id); m_logtype_dict_entry.add_dictionary_var(); @@ -422,21 +421,68 @@ void Archive::write_msg_using_schema(LogEventView const& log_view) { break; } default: { - // Variable string looks like a dictionary variable, so encode it as so - encoded_variable_t encoded_var; - variable_dictionary_id_t id; - m_var_dict.add_entry(token.to_string(), id); - encoded_var = EncodedVariableInterpreter::encode_var_dict_id(id); - m_var_ids.push_back(id); + auto const& lexer{log_view.get_log_parser().m_lexer}; + auto capture_ids{lexer.get_capture_ids_from_rule_id(token_type)}; + + // If the variable token contains capture groups, we break the token up by storing + // each capture as a variable and any substrings surrounding the capture as part of + // the logtype. If there are no capture groups the entire variable token is stored + // as a variable. + if (false == capture_ids.has_value()) { + variable_dictionary_id_t id{}; + m_var_dict.add_entry(token.to_string(), id); + m_var_ids.push_back(id); + m_encoded_vars.push_back(EncodedVariableInterpreter::encode_var_dict_id(id)); + m_logtype_dict_entry.add_dictionary_var(); + + break; + } + auto const register_ids{lexer.get_reg_ids_from_capture_id(capture_ids.value().at(0))}; + if (false == register_ids.has_value()) { + throw(std::runtime_error( + "No register IDs found for variable's capture group. Full token: " + + token.to_string() + )); + } + auto const [start_reg_id, end_reg_id]{register_ids.value()}; + auto const capture_start{token.get_reversed_reg_positions(start_reg_id).front()}; + auto const capture_end{token.get_reversed_reg_positions(end_reg_id).back()}; + auto const token_view{token.to_string_view()}; + size_t token_pos{0}; + + auto const before_capture{token_view.substr(token_pos, capture_start)}; + m_logtype_dict_entry + .add_constant(before_capture, 0, before_capture.length()); + token_pos += before_capture.length(); + + // If a capture has repetition we store all instances as a single variable. + auto const capture_len = [&]() -> size_t { + if (capture_start <= capture_end) { + return capture_end - capture_start; + } + return token_view.length() - capture_start + capture_end; + }(); + auto const capture{token_view.substr(token_pos, capture_len)}; + + variable_dictionary_id_t id{}; + m_var_dict.add_entry(capture, id); + m_var_ids.push_back(id); + m_encoded_vars.push_back( + EncodedVariableInterpreter::encode_var_dict_id(id) + ); m_logtype_dict_entry.add_dictionary_var(); - m_encoded_vars.push_back(encoded_var); + token_pos += capture.length(); + + m_logtype_dict_entry + .add_constant(token_view, token_pos, token_view.length() - token_pos); + break; } } } if (!m_logtype_dict_entry.get_value().empty()) { - logtype_dictionary_id_t logtype_id; + logtype_dictionary_id_t logtype_id{}; m_logtype_dict.add_entry(m_logtype_dict_entry, logtype_id); m_file->write_encoded_msg( timestamp, diff --git a/components/core/tests/test-ParserWithUserSchema.cpp b/components/core/tests/test-ParserWithUserSchema.cpp index 61ca7ffdbe..9cc99efb1f 100644 --- a/components/core/tests/test-ParserWithUserSchema.cpp +++ b/components/core/tests/test-ParserWithUserSchema.cpp @@ -179,3 +179,26 @@ TEST_CASE("Test lexer", "[Search]") { token = opt_token.value(); } } + +TEST_CASE("Test schema with single capture group", "[load_lexer]") { + std::string const schema_path{"../tests/test_schema_files/single_capture_group.txt"}; + ByteLexer lexer; + load_lexer_from_file(schema_path, lexer); + + auto const rule_id{lexer.m_symbol_id.at("capture")}; + auto const capture_ids{lexer.get_capture_ids_from_rule_id(rule_id)}; + REQUIRE(capture_ids.has_value()); + REQUIRE(1 == capture_ids.value().size()); + REQUIRE("group" == lexer.m_id_symbol.at(capture_ids->at(0))); +} + +TEST_CASE("Test error for schema rule with multiple capture groups", "[load_lexer]") { + std::string const schema_path{"../tests/test_schema_files/multiple_capture_groups.txt"}; + ByteLexer lexer; + REQUIRE_THROWS_WITH( + load_lexer_from_file(schema_path, lexer), + schema_path + + ":1: error: the schema rule 'multicapture' has a regex pattern containing > " + "1 capture group.\n" + ); +} diff --git a/components/core/tests/test_schema_files/multiple_capture_groups.txt b/components/core/tests/test_schema_files/multiple_capture_groups.txt new file mode 100644 index 0000000000..e55c56a9be --- /dev/null +++ b/components/core/tests/test_schema_files/multiple_capture_groups.txt @@ -0,0 +1 @@ +multicapture:text(?var0)text(?var1)text diff --git a/components/core/tests/test_schema_files/single_capture_group.txt b/components/core/tests/test_schema_files/single_capture_group.txt new file mode 100644 index 0000000000..6984421a10 --- /dev/null +++ b/components/core/tests/test_schema_files/single_capture_group.txt @@ -0,0 +1 @@ +capture:text(?var)text From 61d5715f41dc26fee19c1c060804de922a12a20f Mon Sep 17 00:00:00 2001 From: davidlion Date: Thu, 28 Aug 2025 11:43:28 -0400 Subject: [PATCH 02/91] Add schema equals rule capture. --- components/core/config/schemas.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index e0b777859d..9c09fb5d45 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -16,4 +16,4 @@ float:\-{0,1}[0-9]+\.[0-9]+ // Dictionary variables hex:[a-fA-F]+ hasNumber:.*\d.* -equals:.*=.*[a-zA-Z0-9].* +equals:.*=(?.*[a-zA-Z0-9].*) From 20b26f9956fe37a30309583c3b77755ee4ec5a75 Mon Sep 17 00:00:00 2001 From: davidlion Date: Thu, 18 Sep 2025 13:22:10 -0400 Subject: [PATCH 03/91] Address some review comments. --- .../clp/streaming_archive/writer/Archive.cpp | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/components/core/src/clp/streaming_archive/writer/Archive.cpp b/components/core/src/clp/streaming_archive/writer/Archive.cpp index b1edd0c9ce..f14c6df87a 100644 --- a/components/core/src/clp/streaming_archive/writer/Archive.cpp +++ b/components/core/src/clp/streaming_archive/writer/Archive.cpp @@ -421,13 +421,13 @@ void Archive::write_msg_using_schema(log_surgeon::LogEventView const& log_view) break; } default: { - auto const& lexer{log_view.get_log_parser().m_lexer}; - auto capture_ids{lexer.get_capture_ids_from_rule_id(token_type)}; - + // If there are no capture groups the entire variable token is stored as a variable. // If the variable token contains capture groups, we break the token up by storing // each capture as a variable and any substrings surrounding the capture as part of - // the logtype. If there are no capture groups the entire variable token is stored - // as a variable. + // the logtype. + + auto const& lexer{log_view.get_log_parser().m_lexer}; + auto capture_ids{lexer.get_capture_ids_from_rule_id(token_type)}; if (false == capture_ids.has_value()) { variable_dictionary_id_t id{}; m_var_dict.add_entry(token.to_string(), id); @@ -438,7 +438,9 @@ void Archive::write_msg_using_schema(log_surgeon::LogEventView const& log_view) break; } - auto const register_ids{lexer.get_reg_ids_from_capture_id(capture_ids.value().at(0))}; + auto const register_ids{ + lexer.get_reg_ids_from_capture_id(capture_ids.value().at(0)) + }; if (false == register_ids.has_value()) { throw(std::runtime_error( "No register IDs found for variable's capture group. Full token: " @@ -446,14 +448,13 @@ void Archive::write_msg_using_schema(log_surgeon::LogEventView const& log_view) )); } auto const [start_reg_id, end_reg_id]{register_ids.value()}; - auto const capture_start{token.get_reversed_reg_positions(start_reg_id).front()}; - auto const capture_end{token.get_reversed_reg_positions(end_reg_id).back()}; + auto const capture_start{token.get_reversed_reg_positions(start_reg_id).back()}; + auto const capture_end{token.get_reversed_reg_positions(end_reg_id).front()}; auto const token_view{token.to_string_view()}; size_t token_pos{0}; auto const before_capture{token_view.substr(token_pos, capture_start)}; - m_logtype_dict_entry - .add_constant(before_capture, 0, before_capture.length()); + m_logtype_dict_entry.add_constant(before_capture, 0, before_capture.length()); token_pos += before_capture.length(); // If a capture has repetition we store all instances as a single variable. @@ -468,9 +469,7 @@ void Archive::write_msg_using_schema(log_surgeon::LogEventView const& log_view) variable_dictionary_id_t id{}; m_var_dict.add_entry(capture, id); m_var_ids.push_back(id); - m_encoded_vars.push_back( - EncodedVariableInterpreter::encode_var_dict_id(id) - ); + m_encoded_vars.push_back(EncodedVariableInterpreter::encode_var_dict_id(id)); m_logtype_dict_entry.add_dictionary_var(); token_pos += capture.length(); @@ -481,7 +480,7 @@ void Archive::write_msg_using_schema(log_surgeon::LogEventView const& log_view) } } } - if (!m_logtype_dict_entry.get_value().empty()) { + if (false == m_logtype_dict_entry.get_value().empty()) { logtype_dictionary_id_t logtype_id{}; m_logtype_dict.add_entry(m_logtype_dict_entry, logtype_id); m_file->write_encoded_msg( From b8089f107617b2627bc272d5f4672119c3890d1f Mon Sep 17 00:00:00 2001 From: davidlion Date: Thu, 18 Sep 2025 13:22:54 -0400 Subject: [PATCH 04/91] Tweak tests. --- components/core/src/clp/Utils.cpp | 10 ++++++---- components/core/tests/test-ParserWithUserSchema.cpp | 6 +++--- .../test_schema_files/multiple_capture_groups.txt | 2 ++ .../tests/test_schema_files/single_capture_group.txt | 2 ++ 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/components/core/src/clp/Utils.cpp b/components/core/src/clp/Utils.cpp index 8fcf17ff26..8f22bda569 100644 --- a/components/core/src/clp/Utils.cpp +++ b/components/core/src/clp/Utils.cpp @@ -191,13 +191,15 @@ load_lexer_from_file(std::string const& schema_file_path, log_surgeon::lexers::B // Currently, we only support at most a single capture group in each variable. If a capture // group is present its match will be treated as the variable rather than the full match. - if (1 < rule->m_regex_ptr->get_subtree_positive_captures().size()) { + auto const num_captures = rule->m_regex_ptr->get_subtree_positive_captures().size(); + if (1 < num_captures) { throw std::runtime_error( schema_file_path + ":" + std::to_string(rule->m_line_num + 1) + ": error: the schema rule '" + rule->m_name - + "' has a regex pattern containing > 1 capture group.\n" - ); - } + + "' has a regex pattern containing > 1 capture groups (found " + + std::to_string(num_captures) + ").\n" + ); + } if ("timestamp" == rule->m_name) { continue; diff --git a/components/core/tests/test-ParserWithUserSchema.cpp b/components/core/tests/test-ParserWithUserSchema.cpp index 9cc99efb1f..d2f5adfd74 100644 --- a/components/core/tests/test-ParserWithUserSchema.cpp +++ b/components/core/tests/test-ParserWithUserSchema.cpp @@ -188,7 +188,7 @@ TEST_CASE("Test schema with single capture group", "[load_lexer]") { auto const rule_id{lexer.m_symbol_id.at("capture")}; auto const capture_ids{lexer.get_capture_ids_from_rule_id(rule_id)}; REQUIRE(capture_ids.has_value()); - REQUIRE(1 == capture_ids.value().size()); + REQUIRE(1 == capture_ids->size()); REQUIRE("group" == lexer.m_id_symbol.at(capture_ids->at(0))); } @@ -198,7 +198,7 @@ TEST_CASE("Test error for schema rule with multiple capture groups", "[load_lexe REQUIRE_THROWS_WITH( load_lexer_from_file(schema_path, lexer), schema_path - + ":1: error: the schema rule 'multicapture' has a regex pattern containing > " - "1 capture group.\n" + + ":3: error: the schema rule 'multicapture' has a regex pattern containing > " + "1 capture groups (found 2).\n" ); } diff --git a/components/core/tests/test_schema_files/multiple_capture_groups.txt b/components/core/tests/test_schema_files/multiple_capture_groups.txt index e55c56a9be..09998bc19d 100644 --- a/components/core/tests/test_schema_files/multiple_capture_groups.txt +++ b/components/core/tests/test_schema_files/multiple_capture_groups.txt @@ -1 +1,3 @@ +delimiters: \r\n + multicapture:text(?var0)text(?var1)text diff --git a/components/core/tests/test_schema_files/single_capture_group.txt b/components/core/tests/test_schema_files/single_capture_group.txt index 6984421a10..7c4bf17688 100644 --- a/components/core/tests/test_schema_files/single_capture_group.txt +++ b/components/core/tests/test_schema_files/single_capture_group.txt @@ -1 +1,3 @@ +delimiters: \r\n + capture:text(?var)text From 98d10885ab8bf3fa92cde98473fa0ef1484013e6 Mon Sep 17 00:00:00 2001 From: davidlion Date: Fri, 19 Sep 2025 10:01:31 -0400 Subject: [PATCH 05/91] Fix wrapped token logic. --- .../clp/streaming_archive/writer/Archive.cpp | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/components/core/src/clp/streaming_archive/writer/Archive.cpp b/components/core/src/clp/streaming_archive/writer/Archive.cpp index f14c6df87a..041b41157d 100644 --- a/components/core/src/clp/streaming_archive/writer/Archive.cpp +++ b/components/core/src/clp/streaming_archive/writer/Archive.cpp @@ -424,7 +424,8 @@ void Archive::write_msg_using_schema(log_surgeon::LogEventView const& log_view) // If there are no capture groups the entire variable token is stored as a variable. // If the variable token contains capture groups, we break the token up by storing // each capture as a variable and any substrings surrounding the capture as part of - // the logtype. + // the logtype. If a capture has repetition we store all instances as a single + // variable. auto const& lexer{log_view.get_log_parser().m_lexer}; auto capture_ids{lexer.get_capture_ids_from_rule_id(token_type)}; @@ -450,31 +451,32 @@ void Archive::write_msg_using_schema(log_surgeon::LogEventView const& log_view) auto const [start_reg_id, end_reg_id]{register_ids.value()}; auto const capture_start{token.get_reversed_reg_positions(start_reg_id).back()}; auto const capture_end{token.get_reversed_reg_positions(end_reg_id).front()}; - auto const token_view{token.to_string_view()}; - size_t token_pos{0}; - - auto const before_capture{token_view.substr(token_pos, capture_start)}; - m_logtype_dict_entry.add_constant(before_capture, 0, before_capture.length()); - token_pos += before_capture.length(); - // If a capture has repetition we store all instances as a single variable. - auto const capture_len = [&]() -> size_t { - if (capture_start <= capture_end) { - return capture_end - capture_start; + auto const get_wrapped_size = [&](size_t start, size_t end) -> size_t { + if (start <= end) { + return end - start; } - return token_view.length() - capture_start + capture_end; - }(); - auto const capture{token_view.substr(token_pos, capture_len)}; + return token.m_buffer_size - start + end; + }; + + auto const token_view{token.to_string_view()}; + auto const before_capture_size{get_wrapped_size(token.m_start_pos, capture_start)}; + m_logtype_dict_entry.add_constant(token_view, 0, before_capture_size); + auto const capture_size{get_wrapped_size(capture_start, capture_end)}; + auto const capture{token_view.substr(before_capture_size, capture_size)}; variable_dictionary_id_t id{}; m_var_dict.add_entry(capture, id); m_var_ids.push_back(id); m_encoded_vars.push_back(EncodedVariableInterpreter::encode_var_dict_id(id)); m_logtype_dict_entry.add_dictionary_var(); - token_pos += capture.length(); - m_logtype_dict_entry - .add_constant(token_view, token_pos, token_view.length() - token_pos); + auto const capture_end_pos{before_capture_size + capture_size}; + m_logtype_dict_entry.add_constant( + token_view, + capture_end_pos, + token_view.length() - capture_end_pos + ); break; } From dec9f6e97d9319b6ade25ee8ebcafb6cb96685e2 Mon Sep 17 00:00:00 2001 From: davidlion Date: Fri, 19 Sep 2025 10:28:46 -0400 Subject: [PATCH 06/91] Fix formatting. --- components/core/src/clp/Utils.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp/Utils.cpp b/components/core/src/clp/Utils.cpp index 8f22bda569..f8449bc4ca 100644 --- a/components/core/src/clp/Utils.cpp +++ b/components/core/src/clp/Utils.cpp @@ -198,8 +198,8 @@ load_lexer_from_file(std::string const& schema_file_path, log_surgeon::lexers::B + ": error: the schema rule '" + rule->m_name + "' has a regex pattern containing > 1 capture groups (found " + std::to_string(num_captures) + ").\n" - ); - } + ); + } if ("timestamp" == rule->m_name) { continue; From 03a1f029a7808bb2273684dce38e8d817cfa0c46 Mon Sep 17 00:00:00 2001 From: davidlion Date: Fri, 19 Sep 2025 10:29:13 -0400 Subject: [PATCH 07/91] Remove wrapping logic by manipulating the token. --- .../clp/streaming_archive/writer/Archive.cpp | 32 ++++++++----------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/components/core/src/clp/streaming_archive/writer/Archive.cpp b/components/core/src/clp/streaming_archive/writer/Archive.cpp index 041b41157d..7d125847ea 100644 --- a/components/core/src/clp/streaming_archive/writer/Archive.cpp +++ b/components/core/src/clp/streaming_archive/writer/Archive.cpp @@ -451,32 +451,26 @@ void Archive::write_msg_using_schema(log_surgeon::LogEventView const& log_view) auto const [start_reg_id, end_reg_id]{register_ids.value()}; auto const capture_start{token.get_reversed_reg_positions(start_reg_id).back()}; auto const capture_end{token.get_reversed_reg_positions(end_reg_id).front()}; + auto token_view{token}; + auto const token_end_pos{token_view.m_end_pos}; - auto const get_wrapped_size = [&](size_t start, size_t end) -> size_t { - if (start <= end) { - return end - start; - } - return token.m_buffer_size - start + end; - }; - - auto const token_view{token.to_string_view()}; - auto const before_capture_size{get_wrapped_size(token.m_start_pos, capture_start)}; - m_logtype_dict_entry.add_constant(token_view, 0, before_capture_size); + token_view.m_end_pos = capture_start; + auto const text_before_capture{token_view.to_string_view()}; + m_logtype_dict_entry + .add_constant(text_before_capture, 0, text_before_capture.size()); - auto const capture_size{get_wrapped_size(capture_start, capture_end)}; - auto const capture{token_view.substr(before_capture_size, capture_size)}; + token_view.m_start_pos = capture_start; + token_view.m_end_pos = capture_end; variable_dictionary_id_t id{}; - m_var_dict.add_entry(capture, id); + m_var_dict.add_entry(token_view.to_string_view(), id); m_var_ids.push_back(id); m_encoded_vars.push_back(EncodedVariableInterpreter::encode_var_dict_id(id)); m_logtype_dict_entry.add_dictionary_var(); - auto const capture_end_pos{before_capture_size + capture_size}; - m_logtype_dict_entry.add_constant( - token_view, - capture_end_pos, - token_view.length() - capture_end_pos - ); + token_view.m_start_pos = capture_end; + token_view.m_end_pos = token_end_pos; + auto const text_after_capture{token_view.to_string_view()}; + m_logtype_dict_entry.add_constant(text_after_capture, 0, text_after_capture.size()); break; } From 65541fbe09a1078418a0d70deb6301d12d416f3e Mon Sep 17 00:00:00 2001 From: davidlion Date: Tue, 18 Nov 2025 21:41:07 -0500 Subject: [PATCH 08/91] Update log-surgeon. --- taskfiles/deps/main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 927cedb2a1..a5a4785b26 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -367,8 +367,8 @@ tasks: - "-DCMAKE_INSTALL_MESSAGE=LAZY" - "-Dlog_surgeon_BUILD_TESTING=OFF" LIB_NAME: "log_surgeon" - TARBALL_SHA256: "4551ea50cd22e8423770fd66a167e1c86053b1f4957f72c582a2da93e7820210" - TARBALL_URL: "https://github.com/y-scope/log-surgeon/archive/840f262.tar.gz" + TARBALL_SHA256: "396ef8822e687b0bb8ca433a360e9e1d39a51620df89cac606db5e0f49d1b265" + TARBALL_URL: "https://github.com/y-scope/log-surgeon/archive/1135c2e.tar.gz" lz4: internal: true From 56bb7a6526d29e10782d27af8379787e47936dc4 Mon Sep 17 00:00:00 2001 From: davidlion Date: Tue, 18 Nov 2025 22:42:07 -0500 Subject: [PATCH 09/91] Update GrepCore. --- components/core/src/clp/GrepCore.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp/GrepCore.cpp b/components/core/src/clp/GrepCore.cpp index 1a4bf499e2..b8abf5a980 100644 --- a/components/core/src/clp/GrepCore.cpp +++ b/components/core/src/clp/GrepCore.cpp @@ -257,9 +257,9 @@ bool GrepCore::get_bounds_of_next_potential_var( return false; } search_token = SearchToken{token.value()}; - search_token.m_type_ids_set.insert(search_token.m_type_ids_ptr->at(0)); + search_token.m_type_ids_set.insert(search_token.get_type_ids()->at(0)); } - auto const& type = search_token.m_type_ids_ptr->at(0); + auto const& type = search_token.get_type_ids()->at(0); if (type != static_cast(log_surgeon::SymbolId::TokenUncaughtString) && type != static_cast(log_surgeon::SymbolId::TokenEnd)) { From 610e76f218201262773d8f2e12ac2b6703517cb1 Mon Sep 17 00:00:00 2001 From: davidlion Date: Wed, 19 Nov 2025 23:03:12 -0500 Subject: [PATCH 10/91] Update log surgeon code + tests with new version. --- .../clp/streaming_archive/writer/Archive.cpp | 109 +++++++++--------- .../core/tests/test-ParserWithUserSchema.cpp | 15 +-- .../tests/test_schema_files/search_schema.txt | 2 +- 3 files changed, 65 insertions(+), 61 deletions(-) diff --git a/components/core/src/clp/streaming_archive/writer/Archive.cpp b/components/core/src/clp/streaming_archive/writer/Archive.cpp index 9934683f5f..ce71ee7a91 100644 --- a/components/core/src/clp/streaming_archive/writer/Archive.cpp +++ b/components/core/src/clp/streaming_archive/writer/Archive.cpp @@ -22,6 +22,7 @@ #include "../../spdlog_with_specializations.hpp" #include "../../Utils.hpp" #include "../Constants.hpp" +#include "TimestampPattern.hpp" #include "utils.hpp" using clp::ir::eight_byte_encoded_variable_t; @@ -315,14 +316,14 @@ Archive::write_msg(epochtime_t timestamp, string const& message, size_t num_unco } void Archive::write_msg_using_schema(log_surgeon::LogEventView const& log_view) { - epochtime_t timestamp = 0; - TimestampPattern* timestamp_pattern = nullptr; - auto const& log_output_buffer = log_view.get_log_output_buffer(); - if (log_output_buffer->has_timestamp()) { + epochtime_t timestamp{0}; + TimestampPattern const* timestamp_pattern{nullptr}; + auto const& log_buf = log_view.get_log_output_buffer(); + if (log_buf->has_timestamp()) { size_t start{}; size_t end{}; - timestamp_pattern = (TimestampPattern*)TimestampPattern::search_known_ts_patterns( - log_output_buffer->get_mutable_token(0).to_string(), + timestamp_pattern = TimestampPattern::search_known_ts_patterns( + log_buf->get_mutable_token(0).to_string(), timestamp, start, end @@ -330,14 +331,16 @@ void Archive::write_msg_using_schema(log_surgeon::LogEventView const& log_view) if (nullptr == timestamp_pattern) { throw(std::runtime_error( "Schema contains a timestamp regex that matches " - + log_output_buffer->get_mutable_token(0).to_string() + + log_buf->get_mutable_token(0).to_string() + " which does not match any known timestamp pattern." )); } if (m_old_ts_pattern != timestamp_pattern) { change_ts_pattern(timestamp_pattern); - m_old_ts_pattern = timestamp_pattern; + m_old_ts_pattern = const_cast(timestamp_pattern); } + } else { + timestamp_pattern = nullptr; } if (get_data_size_of_dictionaries() >= m_target_data_size_of_dicts) { split_file_and_archive( @@ -353,48 +356,45 @@ void Archive::write_msg_using_schema(log_surgeon::LogEventView const& log_view) m_encoded_vars.clear(); m_var_ids.clear(); m_logtype_dict_entry.clear(); - size_t num_uncompressed_bytes = 0; + + size_t num_uncompressed_bytes{0}; // Timestamp is included in the uncompressed message size - uint32_t start_pos = log_output_buffer->get_token(0).m_start_pos; + auto start_pos{log_buf->get_token(0).get_start_pos()}; if (timestamp_pattern == nullptr) { - start_pos = log_output_buffer->get_token(1).m_start_pos; + start_pos = log_buf->get_token(1).get_start_pos(); } - uint32_t const end_pos = log_output_buffer->get_token(log_output_buffer->pos() - 1).m_end_pos; + auto const end_pos{log_buf->get_token(log_buf->pos() - 1).get_end_pos()}; if (start_pos <= end_pos) { num_uncompressed_bytes = end_pos - start_pos; } else { - num_uncompressed_bytes - = log_output_buffer->get_token(0).m_buffer_size - start_pos + end_pos; + num_uncompressed_bytes = log_buf->get_token(0).get_buffer_size() - start_pos + end_pos; } - for (uint32_t i = 1; i < log_output_buffer->pos(); i++) { - log_surgeon::Token& token = log_output_buffer->get_mutable_token(i); - auto const token_type{token.m_type_ids_ptr->at(0)}; - if (log_output_buffer->has_delimiters() && (timestamp_pattern != nullptr || i > 1) + for (auto token_idx{1}; token_idx < log_buf->pos(); token_idx++) { + auto token_view{log_buf->get_token(token_idx)}; + auto const token_type{token_view.get_type_ids()->at(0)}; + if (log_buf->has_delimiters() && (timestamp_pattern != nullptr || token_idx > 1) && token_type != static_cast(log_surgeon::SymbolId::TokenUncaughtString) && token_type != static_cast(log_surgeon::SymbolId::TokenNewline)) { - m_logtype_dict_entry.add_constant(token.get_delimiter(), 0, 1); - if (token.m_start_pos == token.m_buffer_size - 1) { - token.m_start_pos = 0; - } else { - token.m_start_pos++; - } + m_logtype_dict_entry.add_constant(token_view.get_delimiter(), 0, 1); + token_view.increment_start_pos(); } switch (token_type) { case static_cast(log_surgeon::SymbolId::TokenNewline): case static_cast(log_surgeon::SymbolId::TokenUncaughtString): { - m_logtype_dict_entry.add_constant(token.to_string(), 0, token.get_length()); + m_logtype_dict_entry + .add_constant(token_view.to_string(), 0, token_view.get_length()); break; } case static_cast(log_surgeon::SymbolId::TokenInt): { encoded_variable_t encoded_var{}; if (!EncodedVariableInterpreter::convert_string_to_representable_integer_var( - token.to_string(), + token_view.to_string(), encoded_var )) { variable_dictionary_id_t id{}; - m_var_dict.add_entry(token.to_string(), id); + m_var_dict.add_entry(token_view.to_string(), id); encoded_var = EncodedVariableInterpreter::encode_var_dict_id(id); m_logtype_dict_entry.add_dictionary_var(); } else { @@ -406,12 +406,12 @@ void Archive::write_msg_using_schema(log_surgeon::LogEventView const& log_view) case static_cast(log_surgeon::SymbolId::TokenFloat): { encoded_variable_t encoded_var{}; if (!EncodedVariableInterpreter::convert_string_to_representable_float_var( - token.to_string(), + token_view.to_string(), encoded_var )) { variable_dictionary_id_t id{}; - m_var_dict.add_entry(token.to_string(), id); + m_var_dict.add_entry(token_view.to_string(), id); encoded_var = EncodedVariableInterpreter::encode_var_dict_id(id); m_logtype_dict_entry.add_dictionary_var(); } else { @@ -424,18 +424,17 @@ void Archive::write_msg_using_schema(log_surgeon::LogEventView const& log_view) // If there are no capture groups the entire variable token is stored as a variable. // If the variable token contains capture groups, we break the token up by storing // each capture as a variable and any substrings surrounding the capture as part of - // the logtype. If a capture has repetition we store all instances as a single - // variable. + // the logtype. Capture repetition currently does not work so we explicitly only + // store the first capture. auto const& lexer{log_view.get_log_parser().m_lexer}; auto capture_ids{lexer.get_capture_ids_from_rule_id(token_type)}; if (false == capture_ids.has_value()) { variable_dictionary_id_t id{}; - m_var_dict.add_entry(token.to_string(), id); + m_var_dict.add_entry(token_view.to_string(), id); m_var_ids.push_back(id); m_encoded_vars.push_back(EncodedVariableInterpreter::encode_var_dict_id(id)); m_logtype_dict_entry.add_dictionary_var(); - break; } @@ -445,33 +444,37 @@ void Archive::write_msg_using_schema(log_surgeon::LogEventView const& log_view) if (false == register_ids.has_value()) { throw(std::runtime_error( "No register IDs found for variable's capture group. Full token: " - + token.to_string() + + token_view.to_string() )); } + auto const [start_reg_id, end_reg_id]{register_ids.value()}; - auto const capture_start{token.get_reversed_reg_positions(start_reg_id).back()}; - auto const capture_end{token.get_reversed_reg_positions(end_reg_id).front()}; - auto token_view{token}; - auto const token_end_pos{token_view.m_end_pos}; + auto const start_positions{token_view.get_reversed_reg_positions(start_reg_id)}; + auto const end_positions{token_view.get_reversed_reg_positions(end_reg_id)}; - token_view.m_end_pos = capture_start; - auto const text_before_capture{token_view.to_string_view()}; - m_logtype_dict_entry - .add_constant(text_before_capture, 0, text_before_capture.size()); + if (false == start_positions.empty() && -1 < start_positions[0] + && false == end_positions.empty() && -1 < end_positions[0]) + { + auto token_end{token_view.get_end_pos()}; - token_view.m_start_pos = capture_start; - token_view.m_end_pos = capture_end; - variable_dictionary_id_t id{}; - m_var_dict.add_entry(token_view.to_string_view(), id); - m_var_ids.push_back(id); - m_encoded_vars.push_back(EncodedVariableInterpreter::encode_var_dict_id(id)); - m_logtype_dict_entry.add_dictionary_var(); + token_view.set_end_pos(start_positions[0]); + auto const before_capture{token_view.to_string_view()}; + m_logtype_dict_entry.add_constant(before_capture, 0, before_capture.size()); - token_view.m_start_pos = capture_end; - token_view.m_end_pos = token_end_pos; - auto const text_after_capture{token_view.to_string_view()}; - m_logtype_dict_entry.add_constant(text_after_capture, 0, text_after_capture.size()); + token_view.set_start_pos(start_positions[0]); + token_view.set_end_pos(end_positions[0]); + variable_dictionary_id_t id{}; + m_var_dict.add_entry(token_view.to_string_view(), id); + m_var_ids.push_back(id); + m_encoded_vars.push_back(EncodedVariableInterpreter::encode_var_dict_id(id)); + m_logtype_dict_entry.add_dictionary_var(); + + token_view.set_start_pos(end_positions[0]); + token_view.set_end_pos(token_end); + auto const after_capture{token_view.to_string_view()}; + m_logtype_dict_entry.add_constant(after_capture, 0, after_capture.size()); + } break; } } diff --git a/components/core/tests/test-ParserWithUserSchema.cpp b/components/core/tests/test-ParserWithUserSchema.cpp index 491b4b1c79..a18ff00f28 100644 --- a/components/core/tests/test-ParserWithUserSchema.cpp +++ b/components/core/tests/test-ParserWithUserSchema.cpp @@ -192,10 +192,11 @@ TEST_CASE("Test lexer", "[Search]") { auto [error_code, opt_token] = lexer.scan(parser_input_buffer); REQUIRE(error_code == log_surgeon::ErrorCode::Success); Token token{opt_token.value()}; - while (token.m_type_ids_ptr->at(0) != static_cast(log_surgeon::SymbolId::TokenEnd)) { + while (token.get_type_ids()->at(0) != static_cast(log_surgeon::SymbolId::TokenEnd)) { SPDLOG_INFO("token:" + token.to_string() + "\n"); SPDLOG_INFO( - "token.m_type_ids->back():" + lexer.m_id_symbol[token.m_type_ids_ptr->back()] + "\n" + "token.get_type_ids()->back():" + lexer.m_id_symbol[token.get_type_ids()->back()] + + "\n" ); auto [error_code, opt_token] = lexer.scan(parser_input_buffer); REQUIRE(error_code == log_surgeon::ErrorCode::Success); @@ -204,9 +205,9 @@ TEST_CASE("Test lexer", "[Search]") { } TEST_CASE("Test schema with single capture group", "[load_lexer]") { - std::string const schema_path{"../tests/test_schema_files/single_capture_group.txt"}; + auto const schema_file_path{get_test_schema_files_dir() / "single_capture_group.txt"}; ByteLexer lexer; - load_lexer_from_file(schema_path, lexer); + load_lexer_from_file(schema_file_path, lexer); auto const rule_id{lexer.m_symbol_id.at("capture")}; auto const capture_ids{lexer.get_capture_ids_from_rule_id(rule_id)}; @@ -216,11 +217,11 @@ TEST_CASE("Test schema with single capture group", "[load_lexer]") { } TEST_CASE("Test error for schema rule with multiple capture groups", "[load_lexer]") { - std::string const schema_path{"../tests/test_schema_files/multiple_capture_groups.txt"}; + auto const schema_file_path{get_test_schema_files_dir() / "multiple_capture_groups.txt"}; ByteLexer lexer; REQUIRE_THROWS_WITH( - load_lexer_from_file(schema_path, lexer), - schema_path + load_lexer_from_file(schema_file_path, lexer), + schema_file_path.string() + ":3: error: the schema rule 'multicapture' has a regex pattern containing > " "1 capture groups (found 2).\n" ); diff --git a/components/core/tests/test_schema_files/search_schema.txt b/components/core/tests/test_schema_files/search_schema.txt index 60d0c12f00..f49a6dbfa4 100644 --- a/components/core/tests/test_schema_files/search_schema.txt +++ b/components/core/tests/test_schema_files/search_schema.txt @@ -1,5 +1,5 @@ // Delimiters -delimiters: \r\n:,=!;%\? +delimiters: \r\n:,=!;%? // First set of variables timestamp:[0-9]{4}\-[0-9]{2}\-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]{3}){0,1} From 578601a356e3eb029683996d5d28a1022c54985e Mon Sep 17 00:00:00 2001 From: davidlion Date: Mon, 24 Nov 2025 11:43:45 -0500 Subject: [PATCH 11/91] Refactor writing a token to dictionaries into a private helper. --- .../clp/streaming_archive/writer/Archive.cpp | 204 +++++++++--------- .../clp/streaming_archive/writer/Archive.hpp | 12 +- 2 files changed, 115 insertions(+), 101 deletions(-) diff --git a/components/core/src/clp/streaming_archive/writer/Archive.cpp b/components/core/src/clp/streaming_archive/writer/Archive.cpp index ce71ee7a91..19d6223be7 100644 --- a/components/core/src/clp/streaming_archive/writer/Archive.cpp +++ b/components/core/src/clp/streaming_archive/writer/Archive.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include "../../EncodedVariableInterpreter.hpp" @@ -315,6 +316,109 @@ Archive::write_msg(epochtime_t timestamp, string const& message, size_t num_unco update_segment_indices(logtype_id, var_ids); } +auto Archive::add_token_to_dicts( + log_surgeon::LogEventView const& log_view, + log_surgeon::Token token_view +) -> void { + switch (token_view.get_type_ids()->at(0)) { + case static_cast(log_surgeon::SymbolId::TokenNewline): + case static_cast(log_surgeon::SymbolId::TokenUncaughtString): { + m_logtype_dict_entry.add_constant(token_view.to_string(), 0, token_view.get_length()); + break; + } + case static_cast(log_surgeon::SymbolId::TokenInt): { + encoded_variable_t encoded_var{}; + if (!EncodedVariableInterpreter::convert_string_to_representable_integer_var( + token_view.to_string(), + encoded_var + )) + { + variable_dictionary_id_t id{}; + m_var_dict.add_entry(token_view.to_string(), id); + encoded_var = EncodedVariableInterpreter::encode_var_dict_id(id); + m_logtype_dict_entry.add_dictionary_var(); + } else { + m_logtype_dict_entry.add_int_var(); + } + m_encoded_vars.push_back(encoded_var); + break; + } + case static_cast(log_surgeon::SymbolId::TokenFloat): { + encoded_variable_t encoded_var{}; + if (!EncodedVariableInterpreter::convert_string_to_representable_float_var( + token_view.to_string(), + encoded_var + )) + { + variable_dictionary_id_t id{}; + m_var_dict.add_entry(token_view.to_string(), id); + encoded_var = EncodedVariableInterpreter::encode_var_dict_id(id); + m_logtype_dict_entry.add_dictionary_var(); + } else { + m_logtype_dict_entry.add_float_var(); + } + m_encoded_vars.push_back(encoded_var); + break; + } + default: { + // If there are no capture groups the entire variable token is stored as a variable. + // If the variable token contains capture groups, we break the token up by storing + // each capture as a variable and any substrings surrounding the capture as part of + // the logtype. Capture repetition currently does not work so we explicitly only + // store the first capture. + + auto const token_type{token_view.get_type_ids()->at(0)}; + auto const& lexer{log_view.get_log_parser().m_lexer}; + auto capture_ids{lexer.get_capture_ids_from_rule_id(token_type)}; + if (false == capture_ids.has_value()) { + variable_dictionary_id_t id{}; + m_var_dict.add_entry(token_view.to_string(), id); + m_var_ids.push_back(id); + m_encoded_vars.push_back(EncodedVariableInterpreter::encode_var_dict_id(id)); + m_logtype_dict_entry.add_dictionary_var(); + break; + } + + auto const register_ids{lexer.get_reg_ids_from_capture_id(capture_ids.value().at(0))}; + if (false == register_ids.has_value()) { + throw(std::runtime_error( + "No register IDs found for variable's capture group. Full token: " + + token_view.to_string() + )); + } + + auto const [start_reg_id, end_reg_id]{register_ids.value()}; + auto const start_positions{token_view.get_reversed_reg_positions(start_reg_id)}; + auto const end_positions{token_view.get_reversed_reg_positions(end_reg_id)}; + + if (false == start_positions.empty() && -1 < start_positions[0] + && false == end_positions.empty() && -1 < end_positions[0]) + { + auto token_end{token_view.get_end_pos()}; + + token_view.set_end_pos(start_positions[0]); + auto const before_capture{token_view.to_string_view()}; + m_logtype_dict_entry.add_constant(before_capture, 0, before_capture.size()); + + token_view.set_start_pos(start_positions[0]); + token_view.set_end_pos(end_positions[0]); + + variable_dictionary_id_t id{}; + m_var_dict.add_entry(token_view.to_string_view(), id); + m_var_ids.push_back(id); + m_encoded_vars.push_back(EncodedVariableInterpreter::encode_var_dict_id(id)); + m_logtype_dict_entry.add_dictionary_var(); + + token_view.set_start_pos(end_positions[0]); + token_view.set_end_pos(token_end); + auto const after_capture{token_view.to_string_view()}; + m_logtype_dict_entry.add_constant(after_capture, 0, after_capture.size()); + } + break; + } + } +} + void Archive::write_msg_using_schema(log_surgeon::LogEventView const& log_view) { epochtime_t timestamp{0}; TimestampPattern const* timestamp_pattern{nullptr}; @@ -379,105 +483,7 @@ void Archive::write_msg_using_schema(log_surgeon::LogEventView const& log_view) m_logtype_dict_entry.add_constant(token_view.get_delimiter(), 0, 1); token_view.increment_start_pos(); } - switch (token_type) { - case static_cast(log_surgeon::SymbolId::TokenNewline): - case static_cast(log_surgeon::SymbolId::TokenUncaughtString): { - m_logtype_dict_entry - .add_constant(token_view.to_string(), 0, token_view.get_length()); - break; - } - case static_cast(log_surgeon::SymbolId::TokenInt): { - encoded_variable_t encoded_var{}; - if (!EncodedVariableInterpreter::convert_string_to_representable_integer_var( - token_view.to_string(), - encoded_var - )) - { - variable_dictionary_id_t id{}; - m_var_dict.add_entry(token_view.to_string(), id); - encoded_var = EncodedVariableInterpreter::encode_var_dict_id(id); - m_logtype_dict_entry.add_dictionary_var(); - } else { - m_logtype_dict_entry.add_int_var(); - } - m_encoded_vars.push_back(encoded_var); - break; - } - case static_cast(log_surgeon::SymbolId::TokenFloat): { - encoded_variable_t encoded_var{}; - if (!EncodedVariableInterpreter::convert_string_to_representable_float_var( - token_view.to_string(), - encoded_var - )) - { - variable_dictionary_id_t id{}; - m_var_dict.add_entry(token_view.to_string(), id); - encoded_var = EncodedVariableInterpreter::encode_var_dict_id(id); - m_logtype_dict_entry.add_dictionary_var(); - } else { - m_logtype_dict_entry.add_float_var(); - } - m_encoded_vars.push_back(encoded_var); - break; - } - default: { - // If there are no capture groups the entire variable token is stored as a variable. - // If the variable token contains capture groups, we break the token up by storing - // each capture as a variable and any substrings surrounding the capture as part of - // the logtype. Capture repetition currently does not work so we explicitly only - // store the first capture. - - auto const& lexer{log_view.get_log_parser().m_lexer}; - auto capture_ids{lexer.get_capture_ids_from_rule_id(token_type)}; - if (false == capture_ids.has_value()) { - variable_dictionary_id_t id{}; - m_var_dict.add_entry(token_view.to_string(), id); - m_var_ids.push_back(id); - m_encoded_vars.push_back(EncodedVariableInterpreter::encode_var_dict_id(id)); - m_logtype_dict_entry.add_dictionary_var(); - break; - } - - auto const register_ids{ - lexer.get_reg_ids_from_capture_id(capture_ids.value().at(0)) - }; - if (false == register_ids.has_value()) { - throw(std::runtime_error( - "No register IDs found for variable's capture group. Full token: " - + token_view.to_string() - )); - } - - auto const [start_reg_id, end_reg_id]{register_ids.value()}; - auto const start_positions{token_view.get_reversed_reg_positions(start_reg_id)}; - auto const end_positions{token_view.get_reversed_reg_positions(end_reg_id)}; - - if (false == start_positions.empty() && -1 < start_positions[0] - && false == end_positions.empty() && -1 < end_positions[0]) - { - auto token_end{token_view.get_end_pos()}; - - token_view.set_end_pos(start_positions[0]); - auto const before_capture{token_view.to_string_view()}; - m_logtype_dict_entry.add_constant(before_capture, 0, before_capture.size()); - - token_view.set_start_pos(start_positions[0]); - token_view.set_end_pos(end_positions[0]); - - variable_dictionary_id_t id{}; - m_var_dict.add_entry(token_view.to_string_view(), id); - m_var_ids.push_back(id); - m_encoded_vars.push_back(EncodedVariableInterpreter::encode_var_dict_id(id)); - m_logtype_dict_entry.add_dictionary_var(); - - token_view.set_start_pos(end_positions[0]); - token_view.set_end_pos(token_end); - auto const after_capture{token_view.to_string_view()}; - m_logtype_dict_entry.add_constant(after_capture, 0, after_capture.size()); - } - break; - } - } + add_token_to_dicts(log_view, token_view); } if (false == m_logtype_dict_entry.get_value().empty()) { logtype_dictionary_id_t logtype_id{}; diff --git a/components/core/src/clp/streaming_archive/writer/Archive.hpp b/components/core/src/clp/streaming_archive/writer/Archive.hpp index 2b589881a5..2fa1d4661f 100644 --- a/components/core/src/clp/streaming_archive/writer/Archive.hpp +++ b/components/core/src/clp/streaming_archive/writer/Archive.hpp @@ -12,7 +12,6 @@ #include #include #include -#include #include "../../ArrayBackedPosIntSet.hpp" #include "../../ErrorCode.hpp" @@ -150,7 +149,7 @@ class Archive { * @param log_event_view * @throw FileWriter::OperationFailed if any write fails */ - void write_msg_using_schema(log_surgeon::LogEventView const& log_event_view); + void write_msg_using_schema(log_surgeon::LogEventView const& log_view); /** * Writes an IR log event to the current encoded file @@ -290,6 +289,15 @@ class Archive { */ auto update_global_metadata() -> void; + /** + * Inspect a log surgeon token and add its information to the logtype and variable dictionaries. + * @param log_view The log event containing the token. + * @param token_view The token to add to the dictionaries. + */ + auto + add_token_to_dicts(log_surgeon::LogEventView const& log_view, log_surgeon::Token token_view) + -> void; + // Variables boost::uuids::uuid m_id; std::string m_id_as_string; From 834fdb780e83ef23f472acb407cc5dcac595908b Mon Sep 17 00:00:00 2001 From: davidlion Date: Wed, 26 Nov 2025 23:19:48 -0500 Subject: [PATCH 12/91] Add unit test; refactor test-ParserWithUserSchema.cpp. --- .../core/tests/test-ParserWithUserSchema.cpp | 164 ++++++++++-------- .../tests/test_log_files/log_with_capture.txt | 1 + .../single_capture_group.txt | 2 +- 3 files changed, 90 insertions(+), 77 deletions(-) create mode 100644 components/core/tests/test_log_files/log_with_capture.txt diff --git a/components/core/tests/test-ParserWithUserSchema.cpp b/components/core/tests/test-ParserWithUserSchema.cpp index a18ff00f28..b51d212f9a 100644 --- a/components/core/tests/test-ParserWithUserSchema.cpp +++ b/components/core/tests/test-ParserWithUserSchema.cpp @@ -1,37 +1,49 @@ // TODO: move this test to log_surgeon // TODO: move load_lexer_from_file into SearchParser in log_surgeon -#include - #include #include +#include #include +#include #include #include +#include +#include #include #include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include -#include "../src/clp/clp/run.hpp" -#include "../src/clp/GlobalMySQLMetadataDB.hpp" -#include "../src/clp/LogSurgeonReader.hpp" -#include "../src/clp/Utils.hpp" +#include "TestOutputCleaner.hpp" -using clp::FileReader; using clp::load_lexer_from_file; -using clp::LogSurgeonReader; -using log_surgeon::DelimiterStringAST; using log_surgeon::lexers::ByteLexer; using log_surgeon::LogParser; -using log_surgeon::ParserAST; using log_surgeon::SchemaAST; -using log_surgeon::SchemaVarAST; -using log_surgeon::Token; namespace { +constexpr std::string_view cTestArchiveDirectory{"test-parser-with-user-schema-archive"}; + +auto run_clp_compress( + std::filesystem::path const& schema_path, + std::filesystem::path const& output_path, + std::filesystem::path const& input_path +) -> int; [[nodiscard]] auto get_tests_dir() -> std::filesystem::path; [[nodiscard]] auto get_test_schema_files_dir() -> std::filesystem::path; [[nodiscard]] auto get_test_queries_dir() -> std::filesystem::path; +[[nodiscard]] auto get_test_log_dir() -> std::filesystem::path; auto get_tests_dir() -> std::filesystem::path { std::filesystem::path const current_file_path{__FILE__}; @@ -45,6 +57,30 @@ auto get_test_schema_files_dir() -> std::filesystem::path { auto get_test_queries_dir() -> std::filesystem::path { return get_tests_dir() / "test_search_queries"; } + +auto get_test_log_dir() -> std::filesystem::path { + return get_tests_dir() / "test_log_files"; +} + +auto run_clp_compress( + std::filesystem::path const& schema_path, + std::filesystem::path const& output_path, + std::filesystem::path const& input_path +) -> int { + auto const schema_path_str{schema_path.string()}; + auto const output_path_str{output_path.string()}; + auto const input_path_str{input_path.string()}; + std::vector argv{ + "clp", + "c", + "--schema-path", + schema_path_str.data(), + output_path_str.data(), + input_path_str.data(), + nullptr + }; + return clp::clp::run(static_cast(argv.size() - 1), argv.data()); +} } // namespace std::unique_ptr generate_schema_ast(std::string const& schema_file) { @@ -60,43 +96,6 @@ std::unique_ptr generate_log_parser(std::string const& schema_file) { return log_parser; } -void compress( - std::string const& output_dir, - std::string const& file_to_compress, - std::string schema_file, - bool old = false -) { - std::vector arguments; - if (old) { - arguments = {"main.cpp", "c", output_dir, file_to_compress}; - } else { - arguments - = {"main.cpp", - "c", - output_dir, - file_to_compress, - "--schema-path", - std::move(schema_file)}; - } - std::vector argv; - for (auto const& arg : arguments) { - argv.push_back(arg.data()); - } - argv.push_back(nullptr); - clp::clp::run(argv.size() - 1, argv.data()); -} - -void decompress(std::string archive_dir, std::string output_dir) { - std::vector arguments - = {"main.cpp", "x", std::move(archive_dir), std::move(output_dir)}; - std::vector argv; - for (auto const& arg : arguments) { - argv.push_back(arg.data()); - } - argv.push_back(nullptr); - clp::clp::run(argv.size() - 1, argv.data()); -} - TEST_CASE("Test error for missing schema file", "[LALR1Parser][SchemaParser]") { auto const file_path = get_test_schema_files_dir() / "missing_schema.txt"; auto const file_path_string = file_path.string(); @@ -156,42 +155,19 @@ TEST_CASE("Test creating log parser without delimiters", "[LALR1Parser][LogParse ); } -// TODO: This test doesn't currently work because delimiters are allowed in -// schema files, and there is no option to disable this yet -// TEST_CASE("Test error for creating log file with delimiter in regex pattern", -// "[LALR1Parser]SchemaParser]") { -// std::string file_path = "../tests/test_schema_files/schema_with_delimiter_in_regex_error.txt"; -// std::string file_name = boost::filesystem::canonical(file_path).string(); -// REQUIRE_THROWS_WITH(generate_log_parser(file_path), -// file_name + -// ":2: error: 'equals' has regex pattern which contains delimiter '='.\n" -// + " equals:.*=.*\n" -// + " ^^^^^\n"); -//} - -// TODO: This error check is performed correctly by CLP, but it is handled by -// something different now so this test will fail as is -// TEST_CASE("Test error for missing log file", "[LALR1Parser][LogParser]") { -// std::string file_name = "../tests/test_log_files/missing_log.txt"; -// std::string file_path = boost::filesystem::weakly_canonical(file_name).string(); -// REQUIRE_THROWS(compress("../tests/test_archives", file_name, -// "../tests/test_schema_files/schema_that_does_not_exist.txt"), -// "Specified schema file does not exist."); -//} - TEST_CASE("Test lexer", "[Search]") { ByteLexer lexer; auto const schema_file_path = get_test_schema_files_dir() / "search_schema.txt"; load_lexer_from_file(schema_file_path.string(), lexer); auto const query_file_path = get_test_queries_dir() / "easy.txt"; - FileReader file_reader{query_file_path.string()}; - LogSurgeonReader reader_wrapper(file_reader); + clp::FileReader file_reader{query_file_path.string()}; + clp::LogSurgeonReader reader_wrapper(file_reader); log_surgeon::ParserInputBuffer parser_input_buffer; parser_input_buffer.read_if_safe(reader_wrapper); lexer.reset(); auto [error_code, opt_token] = lexer.scan(parser_input_buffer); REQUIRE(error_code == log_surgeon::ErrorCode::Success); - Token token{opt_token.value()}; + auto token{opt_token.value()}; while (token.get_type_ids()->at(0) != static_cast(log_surgeon::SymbolId::TokenEnd)) { SPDLOG_INFO("token:" + token.to_string() + "\n"); SPDLOG_INFO( @@ -216,7 +192,7 @@ TEST_CASE("Test schema with single capture group", "[load_lexer]") { REQUIRE("group" == lexer.m_id_symbol.at(capture_ids->at(0))); } -TEST_CASE("Test error for schema rule with multiple capture groups", "[load_lexer]") { +TEST_CASE("Error on schema rule with multiple capture groups", "[load_lexer]") { auto const schema_file_path{get_test_schema_files_dir() / "multiple_capture_groups.txt"}; ByteLexer lexer; REQUIRE_THROWS_WITH( @@ -226,3 +202,39 @@ TEST_CASE("Test error for schema rule with multiple capture groups", "[load_lexe "1 capture groups (found 2).\n" ); } + +TEST_CASE("Verify dictionary contents", "[Compression]") { + auto const log_file_path{get_test_log_dir() / "log_with_capture.txt"}; + auto const schema_file_path{get_test_schema_files_dir() / "single_capture_group.txt"}; + TestOutputCleaner const cleaner{{std::string{cTestArchiveDirectory}}}; + std::filesystem::create_directory(cTestArchiveDirectory); + + REQUIRE(0 == run_clp_compress(schema_file_path, cTestArchiveDirectory, log_file_path)); + + std::vector archive_paths; + for (auto const& entry : std::filesystem::directory_iterator{cTestArchiveDirectory}) { + auto const& path{entry.path()}; + if (false == path.string().ends_with(clp::streaming_archive::cMetadataDBFileName)) { + archive_paths.emplace_back(path); + } + } + REQUIRE(1 == archive_paths.size()); + + clp::streaming_archive::reader::Archive archive_reader; + archive_reader.open(archive_paths.at(0)); + archive_reader.refresh_dictionaries(); + + auto const& logtype_dict{archive_reader.get_logtype_dictionary()}; + REQUIRE(1 == logtype_dict.get_entries().size()); + REQUIRE(fmt::format( + "2016-05-08 07:34:05.251 MyDog{} APet{} test.txt\n", + clp::enum_to_underlying_type(clp::ir::VariablePlaceholder::Dictionary), + clp::enum_to_underlying_type(clp::ir::VariablePlaceholder::Dictionary) + ) + == logtype_dict.get_value(0)); + + auto const& var_dict{archive_reader.get_var_dictionary()}; + REQUIRE(2 == var_dict.get_entries().size()); + REQUIRE("123" == var_dict.get_value(0)); + REQUIRE("4123" == var_dict.get_value(1)); +} diff --git a/components/core/tests/test_log_files/log_with_capture.txt b/components/core/tests/test_log_files/log_with_capture.txt new file mode 100644 index 0000000000..6210ea8bdc --- /dev/null +++ b/components/core/tests/test_log_files/log_with_capture.txt @@ -0,0 +1 @@ +2016-05-08 07:34:05.251 MyDog123 APet4123 test.txt diff --git a/components/core/tests/test_schema_files/single_capture_group.txt b/components/core/tests/test_schema_files/single_capture_group.txt index 7c4bf17688..b9543fbdcc 100644 --- a/components/core/tests/test_schema_files/single_capture_group.txt +++ b/components/core/tests/test_schema_files/single_capture_group.txt @@ -1,3 +1,3 @@ delimiters: \r\n -capture:text(?var)text +capture:[A-Za-z]+(?\d+) From ea1ad3b63b22a1ddb5418b3bd0b03429cca9b6e0 Mon Sep 17 00:00:00 2001 From: davidlion Date: Thu, 27 Nov 2025 09:14:31 -0500 Subject: [PATCH 13/91] Add null + empty check for token type ids. --- .../src/clp/streaming_archive/writer/Archive.cpp | 12 ++++++++++-- .../src/clp/streaming_archive/writer/Archive.hpp | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/components/core/src/clp/streaming_archive/writer/Archive.cpp b/components/core/src/clp/streaming_archive/writer/Archive.cpp index 19d6223be7..194c55488e 100644 --- a/components/core/src/clp/streaming_archive/writer/Archive.cpp +++ b/components/core/src/clp/streaming_archive/writer/Archive.cpp @@ -320,7 +320,11 @@ auto Archive::add_token_to_dicts( log_surgeon::LogEventView const& log_view, log_surgeon::Token token_view ) -> void { - switch (token_view.get_type_ids()->at(0)) { + auto const* type_ids{token_view.get_type_ids()}; + if (nullptr == type_ids || type_ids->empty()) { + throw std::runtime_error("Token has no type IDs: " + token_view.to_string()); + } + switch (type_ids->at(0)) { case static_cast(log_surgeon::SymbolId::TokenNewline): case static_cast(log_surgeon::SymbolId::TokenUncaughtString): { m_logtype_dict_entry.add_constant(token_view.to_string(), 0, token_view.get_length()); @@ -475,7 +479,11 @@ void Archive::write_msg_using_schema(log_surgeon::LogEventView const& log_view) } for (auto token_idx{1}; token_idx < log_buf->pos(); token_idx++) { auto token_view{log_buf->get_token(token_idx)}; - auto const token_type{token_view.get_type_ids()->at(0)}; + auto const* type_ids{token_view.get_type_ids()}; + if (nullptr == type_ids || type_ids->empty()) { + throw std::runtime_error("Token has no type IDs: " + token_view.to_string()); + } + auto const token_type{type_ids->at(0)}; if (log_buf->has_delimiters() && (timestamp_pattern != nullptr || token_idx > 1) && token_type != static_cast(log_surgeon::SymbolId::TokenUncaughtString) && token_type != static_cast(log_surgeon::SymbolId::TokenNewline)) diff --git a/components/core/src/clp/streaming_archive/writer/Archive.hpp b/components/core/src/clp/streaming_archive/writer/Archive.hpp index 2fa1d4661f..b1351199a2 100644 --- a/components/core/src/clp/streaming_archive/writer/Archive.hpp +++ b/components/core/src/clp/streaming_archive/writer/Archive.hpp @@ -149,7 +149,7 @@ class Archive { * @param log_event_view * @throw FileWriter::OperationFailed if any write fails */ - void write_msg_using_schema(log_surgeon::LogEventView const& log_view); + auto write_msg_using_schema(log_surgeon::LogEventView const& log_view) -> void; /** * Writes an IR log event to the current encoded file From 3b36328baba7de064484e6af5bf839d63ee0e43a Mon Sep 17 00:00:00 2001 From: davidlion Date: Thu, 27 Nov 2025 09:25:27 -0500 Subject: [PATCH 14/91] Fix coderabbit nits. --- .../core/src/clp/streaming_archive/writer/Archive.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/components/core/src/clp/streaming_archive/writer/Archive.cpp b/components/core/src/clp/streaming_archive/writer/Archive.cpp index 194c55488e..dcc191f5bd 100644 --- a/components/core/src/clp/streaming_archive/writer/Archive.cpp +++ b/components/core/src/clp/streaming_archive/writer/Archive.cpp @@ -324,7 +324,8 @@ auto Archive::add_token_to_dicts( if (nullptr == type_ids || type_ids->empty()) { throw std::runtime_error("Token has no type IDs: " + token_view.to_string()); } - switch (type_ids->at(0)) { + auto const token_type{type_ids->at(0)}; + switch (token_type) { case static_cast(log_surgeon::SymbolId::TokenNewline): case static_cast(log_surgeon::SymbolId::TokenUncaughtString): { m_logtype_dict_entry.add_constant(token_view.to_string(), 0, token_view.get_length()); @@ -332,7 +333,8 @@ auto Archive::add_token_to_dicts( } case static_cast(log_surgeon::SymbolId::TokenInt): { encoded_variable_t encoded_var{}; - if (!EncodedVariableInterpreter::convert_string_to_representable_integer_var( + if (false + == EncodedVariableInterpreter::convert_string_to_representable_integer_var( token_view.to_string(), encoded_var )) @@ -349,7 +351,8 @@ auto Archive::add_token_to_dicts( } case static_cast(log_surgeon::SymbolId::TokenFloat): { encoded_variable_t encoded_var{}; - if (!EncodedVariableInterpreter::convert_string_to_representable_float_var( + if (false + == EncodedVariableInterpreter::convert_string_to_representable_float_var( token_view.to_string(), encoded_var )) @@ -371,7 +374,6 @@ auto Archive::add_token_to_dicts( // the logtype. Capture repetition currently does not work so we explicitly only // store the first capture. - auto const token_type{token_view.get_type_ids()->at(0)}; auto const& lexer{log_view.get_log_parser().m_lexer}; auto capture_ids{lexer.get_capture_ids_from_rule_id(token_type)}; if (false == capture_ids.has_value()) { From d390b9bfae65eb17d6c34c67029f4766beed824c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 10:41:49 -0500 Subject: [PATCH 15/91] feat(deployment): Add `redis`, `queue` (RabbitMQ), and `results-cache` (MongoDB) components as `StatefulSet`s into the CLP Package Helm chart. (#1698) --- tools/deployment/package-helm/Chart.yaml | 2 +- .../package-helm/templates/_helpers.tpl | 6 +- .../package-helm/templates/configmap.yaml | 29 ++++++ .../templates/database-statefulset.yaml | 19 ++-- .../package-helm/templates/queue-logs-pv.yaml | 9 ++ .../package-helm/templates/queue-secret.yaml | 11 +++ .../package-helm/templates/queue-service.yaml | 15 +++ .../templates/queue-statefulset.yaml | 69 ++++++++++++++ .../package-helm/templates/redis-data-pv.yaml | 9 ++ .../package-helm/templates/redis-logs-pv.yaml | 9 ++ .../package-helm/templates/redis-secret.yaml | 10 ++ .../package-helm/templates/redis-service.yaml | 15 +++ .../templates/redis-statefulset.yaml | 95 +++++++++++++++++++ .../templates/results-cache-data-pv.yaml | 9 ++ .../results-cache-indices-creator-job.yaml | 44 +++++++++ .../templates/results-cache-logs-pv.yaml | 9 ++ .../templates/results-cache-service.yaml | 22 +++++ .../templates/results-cache-statefulset.yaml | 87 +++++++++++++++++ tools/deployment/package-helm/test.sh | 5 +- tools/deployment/package-helm/values.yaml | 18 +++- 20 files changed, 474 insertions(+), 18 deletions(-) create mode 100644 tools/deployment/package-helm/templates/queue-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/queue-secret.yaml create mode 100644 tools/deployment/package-helm/templates/queue-service.yaml create mode 100644 tools/deployment/package-helm/templates/queue-statefulset.yaml create mode 100644 tools/deployment/package-helm/templates/redis-data-pv.yaml create mode 100644 tools/deployment/package-helm/templates/redis-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/redis-secret.yaml create mode 100644 tools/deployment/package-helm/templates/redis-service.yaml create mode 100644 tools/deployment/package-helm/templates/redis-statefulset.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-data-pv.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-service.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-statefulset.yaml diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index 459c4e5bfd..9469a9b469 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "clp" -version: "0.1.2-dev.1" +version: "0.1.2-dev.2" description: "A Helm chart for CLP's (Compressed Log Processor) package deployment" type: "application" appVersion: "0.7.1-dev" diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 92b6eae5d9..6577538785 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -100,12 +100,12 @@ Creates a local PersistentVolume. @param {object} root Root template context @param {string} name PV name @param {string} component Component label -@param {string} capacity Storage capacity -@param {string[]} accessModes Access modes (list) -@param {string} hostPath Absolute path on host @param {string} nodeRole Node role for affinity. Targets nodes with label "node-role.kubernetes.io/". Always falls back to "node-role.kubernetes.io/control-plane" +@param {string} capacity Storage capacity +@param {string[]} accessModes Access modes +@param {string} hostPath Absolute path on host @return {string} YAML-formatted PersistentVolume resource */}} {{- define "clp.createLocalPv" -}} diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 6925e5e174..059d45333c 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -19,6 +19,19 @@ data: package: query_engine: {{ .Values.clpConfig.package.query_engine | quote }} storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} + queue: + host: {{ include "clp.fullname" . }}-queue + port: 5672 + redis: + compression_backend_database: {{ .Values.clpConfig.redis.compression_backend_database }} + host: {{ include "clp.fullname" . }}-redis + port: 6379 + query_backend_database: {{ .Values.clpConfig.redis.query_backend_database }} + results_cache: + db_name: {{ .Values.clpConfig.results_cache.db_name | quote }} + host: {{ include "clp.fullname" . }}-results-cache + port: 27017 + stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} mysql-logging.cnf: | [mysqld] @@ -31,3 +44,19 @@ data: # https://dev.mysql.com/doc/refman/8.0/en/error-log-destination-configuration.html # Send error logs to file log_error=/var/log/mysql/mysql-error.log + + redis.conf: | + # For full reference of configuration options see + # https://raw.githubusercontent.com/redis/redis/7.2/redis.conf + + databases 2 + logfile "/var/log/redis/redis.log" + + mongod.conf: | + replication: + replSetName: "rs0" + systemLog: + destination: file + path: /var/log/mongodb/mongod.log + logAppend: true + timeStampFormat: iso8601-local diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 06347e5062..39fefe81d2 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -64,19 +64,16 @@ spec: readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: &database-health-check - command: - - "sh" - - "-c" - - >- - mysqladmin ping - --host=127.0.0.1 - --port=3306 - --user=${MYSQL_USER} - --password=${MYSQL_PASSWORD} + command: [ + "mysqladmin", "ping", + "--host=127.0.0.1", + "--port=3306", + "--user=$(MYSQL_USER)", + "--password=$(MYSQL_PASSWORD)" + ] livenessProbe: {{- include "clp.livenessProbeTimings" . | nindent 12 }} - exec: - <<: *database-health-check + exec: *database-health-check volumes: - name: "config" configMap: diff --git a/tools/deployment/package-helm/templates/queue-logs-pv.yaml b/tools/deployment/package-helm/templates/queue-logs-pv.yaml new file mode 100644 index 0000000000..cbc4fa3008 --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-logs-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-queue-logs" (include "clp.fullname" .)) + "component" "queue" + "nodeRole" "control-plane" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/queue" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/queue-secret.yaml b/tools/deployment/package-helm/templates/queue-secret.yaml new file mode 100644 index 0000000000..704e443b64 --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: "v1" +kind: "Secret" +metadata: + name: {{ include "clp.fullname" . }}-queue + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "queue" +type: "Opaque" +stringData: + username: {{ .Values.credentials.queue.username | quote }} + password: {{ .Values.credentials.queue.password | quote }} diff --git a/tools/deployment/package-helm/templates/queue-service.yaml b/tools/deployment/package-helm/templates/queue-service.yaml new file mode 100644 index 0000000000..ea357dd2e6 --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-service.yaml @@ -0,0 +1,15 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-queue + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "queue" +spec: + clusterIP: "None" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "queue" + ports: + - port: 5672 + targetPort: "queue" diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml new file mode 100644 index 0000000000..23ae9e39a9 --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -0,0 +1,69 @@ +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp.fullname" . }}-queue + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "queue" +spec: + serviceName: {{ include "clp.fullname" . }}-queue + replicas: 1 + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "queue" + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "queue" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "queue" + image: "rabbitmq:3.9.8" + imagePullPolicy: "Always" + env: + - name: "RABBITMQ_DEFAULT_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-queue + key: "username" + - name: "RABBITMQ_DEFAULT_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-queue + key: "password" + - name: "RABBITMQ_LOGS" + value: "/var/log/rabbitmq/rabbitmq.log" + ports: + - name: "queue" + containerPort: 5672 + volumeMounts: + - name: "queue-logs" + mountPath: "/var/log/rabbitmq" + readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} + exec: &queue-health-check + command: ["rabbitmq-diagnostics", "check_running"] + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + exec: *queue-health-check + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" + volumeClaimTemplates: + - metadata: + name: "queue-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "queue" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/redis-data-pv.yaml b/tools/deployment/package-helm/templates/redis-data-pv.yaml new file mode 100644 index 0000000000..96fa611225 --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-data-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-redis-data" (include "clp.fullname" .)) + "component" "redis" + "nodeRole" "control-plane" + "capacity" "20Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/redis" .Values.clpConfig.data_directory) +) }} diff --git a/tools/deployment/package-helm/templates/redis-logs-pv.yaml b/tools/deployment/package-helm/templates/redis-logs-pv.yaml new file mode 100644 index 0000000000..b3308cbfc9 --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-logs-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-redis-logs" (include "clp.fullname" .)) + "component" "redis" + "nodeRole" "control-plane" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/redis" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/redis-secret.yaml b/tools/deployment/package-helm/templates/redis-secret.yaml new file mode 100644 index 0000000000..244ccb5291 --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-secret.yaml @@ -0,0 +1,10 @@ +apiVersion: "v1" +kind: "Secret" +metadata: + name: {{ include "clp.fullname" . }}-redis + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "redis" +type: "Opaque" +stringData: + password: {{ .Values.credentials.redis.password | quote }} diff --git a/tools/deployment/package-helm/templates/redis-service.yaml b/tools/deployment/package-helm/templates/redis-service.yaml new file mode 100644 index 0000000000..5cd8e61386 --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-service.yaml @@ -0,0 +1,15 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-redis + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "redis" +spec: + clusterIP: "None" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "redis" + ports: + - port: 6379 + targetPort: "redis" diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml new file mode 100644 index 0000000000..d7895ee04a --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -0,0 +1,95 @@ +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp.fullname" . }}-redis + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "redis" +spec: + serviceName: {{ include "clp.fullname" . }}-redis + replicas: 1 + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "redis" + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "redis" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "redis" + image: "redis:7.2.4" + imagePullPolicy: "Always" + command: [ + "redis-server", + "/etc/redis/redis.conf", + "--requirepass", "$(REDIS_PASSWORD)" + ] + env: + - name: "REDIS_PASSWORD" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-redis + key: "password" + ports: + - name: "redis" + containerPort: 6379 + volumeMounts: + - name: "redis-data" + mountPath: "/data" + - name: "redis-logs" + mountPath: "/var/log/redis" + - name: "config" + mountPath: "/etc/redis/redis.conf" + subPath: "redis.conf" + readOnly: true + readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} + exec: &redis-health-check + command: [ + "redis-cli", + "-h", "127.0.0.1", + "-p", "6379", + "-a", "$(REDIS_PASSWORD)", + "PING" + ] + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + exec: *redis-health-check + volumes: + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" + volumeClaimTemplates: + - metadata: + name: "redis-data" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "redis" + resources: + requests: + storage: "10Gi" + - metadata: + name: "redis-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "redis" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml new file mode 100644 index 0000000000..8c67722348 --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-results-cache-data" (include "clp.fullname" .)) + "component" "results-cache" + "nodeRole" "control-plane" + "capacity" "20Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/results-cache" .Values.clpConfig.data_directory) +) }} diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml new file mode 100644 index 0000000000..51c2b822db --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -0,0 +1,44 @@ +apiVersion: "batch/v1" +kind: "Job" +metadata: + name: {{ include "clp.fullname" . }}-results-cache-indices-creator + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache-indices-creator" +spec: + ttlSecondsAfterFinished: 300 + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "results-cache-indices-creator" + spec: + restartPolicy: "OnFailure" + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-results-cache" + image: "busybox:latest" + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-results-cache 27017; + do echo Waiting for results-cache to be ready...; + sleep 2; + done + containers: + - name: "results-cache-indices-creator" + image: "{{ include "clp.image.ref" . }}" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" + env: + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + command: [ + "python3", "-u", + "-m", "clp_py_utils.initialize-results-cache", + "--uri", "mongodb://{{ include "clp.fullname" . }}-results-cache:27017/{{ .Values.clpConfig.results_cache.db_name }}", + "--stream-collection", {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} + ] diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml new file mode 100644 index 0000000000..2050d3a033 --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-results-cache-logs" (include "clp.fullname" .)) + "component" "results-cache" + "nodeRole" "control-plane" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/results-cache" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/results-cache-service.yaml b/tools/deployment/package-helm/templates/results-cache-service.yaml new file mode 100644 index 0000000000..f0503d8bec --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-service.yaml @@ -0,0 +1,22 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-results-cache + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" +spec: + {{- if .Values.allowHostAccessForSbinScripts }} + type: "NodePort" + {{- else }} + clusterIP: "None" + {{- end }} + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" + ports: + - port: 27017 + targetPort: "results-cache" + {{- if .Values.allowHostAccessForSbinScripts }} + nodePort: {{ .Values.clpConfig.results_cache.port }} + {{- end }} diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml new file mode 100644 index 0000000000..adbe0cae9b --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -0,0 +1,87 @@ +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp.fullname" . }}-results-cache + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" +spec: + serviceName: {{ include "clp.fullname" . }}-results-cache + replicas: 1 + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "results-cache" + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "results-cache" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "results-cache" + image: "mongo:7.0.1" + imagePullPolicy: "Always" + args: [ + "--config", "/etc/mongo/mongod.conf", + "--bind_ip", "0.0.0.0" + ] + ports: + - name: "results-cache" + containerPort: 27017 + volumeMounts: + - name: "results-cache-data" + mountPath: "/data/db" + - name: "results-cache-logs" + mountPath: "/var/log/mongodb" + - name: "config" + mountPath: "/etc/mongo/mongod.conf" + subPath: "mongod.conf" + readOnly: true + readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} + exec: &results-cache-health-check + command: + - "mongosh" + - "127.0.0.1:27017/test" + - "--quiet" + - "--eval" + - "db.runCommand('ping').ok" + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + exec: *results-cache-health-check + volumes: + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" + volumeClaimTemplates: + - metadata: + name: "results-cache-data" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "results-cache" + resources: + requests: + storage: "20Gi" + - metadata: + name: "results-cache-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "results-cache" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index c1c45cb9f7..f2f9664170 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -8,7 +8,7 @@ set -o pipefail kind delete cluster --name clp-test rm -rf /tmp/clp -mkdir -p /tmp/clp/var/{data,log}/database +mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache} cat < Date: Mon, 8 Dec 2025 11:13:57 -0500 Subject: [PATCH 16/91] feat(api-server): Add support for steaming query result files from S3. (#1728) Co-authored-by: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> --- Cargo.lock | 1 + components/api-server/Cargo.toml | 1 + components/api-server/src/client.rs | 124 ++++++++++++++++-- components/api-server/src/error.rs | 20 +++ .../src/clp_config/package/config.rs | 7 +- .../src/clp_config/s3_config.rs | 2 +- components/clp-rust-utils/src/s3/client.rs | 10 +- components/clp-rust-utils/src/sqs/client.rs | 11 +- .../log-ingestor/src/aws_client_manager.rs | 22 +--- .../log-ingestor/tests/test_ingestion_job.rs | 6 +- 10 files changed, 161 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be5e17fb22..911d087c25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -107,6 +107,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-stream", + "aws-sdk-s3", "axum", "clap", "clp-rust-utils", diff --git a/components/api-server/Cargo.toml b/components/api-server/Cargo.toml index a4642dabca..fa983ad994 100644 --- a/components/api-server/Cargo.toml +++ b/components/api-server/Cargo.toml @@ -14,6 +14,7 @@ path = "src/bin/api_server.rs" [dependencies] anyhow = "1.0.100" async-stream = "0.3.6" +aws-sdk-s3 = "1.116.0" axum = { version = "0.8.6", features = ["json"] } clap = { version = "4.5.51", features = ["derive"] } clp-rust-utils = { path = "../clp-rust-utils" } diff --git a/components/api-server/src/client.rs b/components/api-server/src/client.rs index d7cf9641bd..59cb303818 100644 --- a/components/api-server/src/client.rs +++ b/components/api-server/src/client.rs @@ -2,9 +2,12 @@ use std::pin::Pin; use async_stream::stream; use clp_rust_utils::{ - clp_config::package::{ - config::{Config, StorageEngine, StreamOutputStorage}, - credentials::Credentials, + clp_config::{ + AwsAuthentication, + package::{ + config::{Config, StorageEngine, StreamOutputStorage}, + credentials::Credentials, + }, }, database::mysql::create_mysql_pool, job_config::{QUERY_JOBS_TABLE_NAME, QueryJobStatus, QueryJobType, SearchJobConfig}, @@ -165,6 +168,7 @@ impl Client { SearchResultStream< impl Stream> + use<>, impl Stream> + use<>, + impl Stream> + use<>, >, ClientError, > { @@ -186,11 +190,19 @@ impl Client { } let job_config = self.get_job_config(search_job_id).await?; + if job_config.write_to_file { - return Ok(SearchResultStream::File { - inner: self.fetch_results_from_file(search_job_id), - }); + let stream = match &self.config.stream_output.storage { + StreamOutputStorage::Fs { .. } => SearchResultStream::File { + inner: self.fetch_results_from_file(search_job_id), + }, + StreamOutputStorage::S3 { .. } => SearchResultStream::S3 { + inner: self.fetch_results_from_s3(search_job_id).await, + }, + }; + return Ok(stream); } + self.fetch_results_from_mongo(search_job_id) .await .map(|s| SearchResultStream::Mongo { inner: s }) @@ -257,7 +269,7 @@ impl Client { search_job_id: u64, ) -> impl Stream> + use<> { let StreamOutputStorage::Fs { directory } = &self.config.stream_output.storage else { - todo!("Outputting query results to S3 is not supported for now."); + unreachable!(); }; let search_job_output_dir = std::path::Path::new("/") .join(directory) @@ -277,6 +289,92 @@ impl Client { stream } + /// Asynchronously fetches results of a completed search job from S3. + /// + /// # Returns + /// + /// A stream of the job's results on success. Each item in the stream is a [`Result`] that: + /// + /// ## Returns + /// + /// The log message in string representation on success. + /// + /// ## Errors + /// + /// Returns an error if: + /// + /// * Forwards the pagination stream's error returned from S3 list-object-v2 operation's + /// paginator (i.e., from + /// [`aws_smithy_async::future::pagination_stream::PaginationStream::next`]). + /// * Forwards S3 get-object operation's return values on failure (i.e., from + /// [`aws_sdk_s3::operation::get_object::builders::GetObjectFluentBuilder::send`]). + /// * Forwards [`aws_smithy_types::byte_stream::ByteStream::collect`]'s return values on + /// failure. + /// + /// # Panics + /// + /// Panics if the stream output storage is not S3. + async fn fetch_results_from_s3( + &self, + search_job_id: u64, + ) -> impl Stream> + use<> { + tracing::info!("Streaming results from S3"); + let StreamOutputStorage::S3 { s3_config, .. } = &self.config.stream_output.storage else { + unreachable!(); + }; + + let AwsAuthentication::Credentials { credentials } = &s3_config.aws_authentication; + + let s3_config = s3_config.clone(); + let credentials = credentials.clone(); + + let s3_client = clp_rust_utils::s3::create_new_client( + s3_config.region_code.as_str(), + credentials.access_key_id.as_str(), + credentials.secret_access_key.as_str(), + None, + ) + .await; + + let key_prefix = format!("{}{}/", s3_config.key_prefix, search_job_id); + tracing::info!("Streaming results from S3 prefix: {}", key_prefix); + let mut object_pages = s3_client + .list_objects_v2() + .bucket(s3_config.bucket.as_str()) + .prefix(key_prefix) + .into_paginator() + .send(); + + stream! { + while let Some(object_page) = object_pages.next().await { + tracing::debug!("Received S3 object page: {:?}", object_page); + for object in object_page?.contents() { + let Some(key) = object.key() else { + tracing::info!("S3 object {:?} doesn't have a key", object); + continue; + }; + if key.ends_with('/') { + tracing::info!("Skipping S3 object with key {} as it is a directory", key); + continue; + } + tracing::info!("Streaming results from S3 object with key: {}", key); + let obj = s3_client + .get_object() + .bucket(s3_config.bucket.as_str()) + .key(key) + .send() + .await?; + let bytes = obj.body.collect().await?.into_bytes(); + let mut deserializer = rmp_serde::Deserializer::from_read_ref(bytes.as_ref()); + while let Ok(event) = Deserialize::deserialize(&mut deserializer) { + let event: (i64, String, String, String, i64) = event; + yield Ok(event.1); + } + } + } + } + } + /// Asynchronously fetches results of a completed search job from `MongoDB`. /// /// # Returns @@ -333,21 +431,26 @@ pin_project! { /// /// * `FileStream`: Streaming from file system storage. /// * `MongoStream`: Streaming from MongoDB storage. + /// * `S3Stream`: Streaming from S3 storage. #[project = SearchResultStreamProj] - pub enum SearchResultStream + pub enum SearchResultStream where FileStream: Stream>, - MongoStream: Stream> + MongoStream: Stream>, + S3Stream: Stream> { File{#[pin] inner: FileStream}, Mongo{#[pin] inner: MongoStream}, + S3{#[pin] inner: S3Stream}, } } -impl Stream for SearchResultStream +impl Stream + for SearchResultStream where FileStream: Stream>, MongoStream: Stream>, + S3Stream: Stream>, { type Item = Result; @@ -358,6 +461,7 @@ where let poll = match self.project() { SearchResultStreamProj::File { inner } => inner.poll_next(cx), SearchResultStreamProj::Mongo { inner } => inner.poll_next(cx), + SearchResultStreamProj::S3 { inner } => inner.poll_next(cx), }; if let std::task::Poll::Ready(Some(Err(err))) = &poll { tracing::error!("An error occurred when streaming results: {}", err); diff --git a/components/api-server/src/error.rs b/components/api-server/src/error.rs index 71526ab142..189d0607fe 100644 --- a/components/api-server/src/error.rs +++ b/components/api-server/src/error.rs @@ -1,3 +1,4 @@ +use aws_sdk_s3::{error::SdkError, primitives::ByteStreamError}; use num_enum::TryFromPrimitive; use thiserror::Error; @@ -17,6 +18,9 @@ pub enum ClientError { #[error("Malformed data")] MalformedData, + + #[error("`AwsError`: {description}")] + Aws { description: String }, } /// Empty trait to mark errors that indicate malformed data. @@ -36,6 +40,22 @@ impl From for ClientError { } } +impl From> for ClientError { + fn from(value: SdkError) -> Self { + Self::Aws { + description: value.to_string(), + } + } +} + +impl From for ClientError { + fn from(value: ByteStreamError) -> Self { + Self::Aws { + description: value.to_string(), + } + } +} + impl From for ClientError { fn from(value: clp_rust_utils::Error) -> Self { match value { diff --git a/components/clp-rust-utils/src/clp_config/package/config.rs b/components/clp-rust-utils/src/clp_config/package/config.rs index a5cb197116..5f1a546e6d 100644 --- a/components/clp-rust-utils/src/clp_config/package/config.rs +++ b/components/clp-rust-utils/src/clp_config/package/config.rs @@ -1,6 +1,6 @@ use serde::Deserialize; -use crate::clp_config::AwsAuthentication; +use crate::clp_config::{AwsAuthentication, S3Config}; /// Mirror of `clp_py_utils.clp_config.ClpConfig`. /// @@ -178,7 +178,10 @@ pub enum StreamOutputStorage { Fs { directory: String }, #[serde(rename = "s3")] - S3 { staging_directory: String }, + S3 { + staging_directory: String, + s3_config: S3Config, + }, } impl Default for StreamOutputStorage { diff --git a/components/clp-rust-utils/src/clp_config/s3_config.rs b/components/clp-rust-utils/src/clp_config/s3_config.rs index 2e49dca3a3..15e70cdae2 100644 --- a/components/clp-rust-utils/src/clp_config/s3_config.rs +++ b/components/clp-rust-utils/src/clp_config/s3_config.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; /// Represents the configuration for connecting to an S3 bucket. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct S3Config { pub bucket: String, pub region_code: String, diff --git a/components/clp-rust-utils/src/s3/client.rs b/components/clp-rust-utils/src/s3/client.rs index a0b522ca5b..337d601e59 100644 --- a/components/clp-rust-utils/src/s3/client.rs +++ b/components/clp-rust-utils/src/s3/client.rs @@ -16,10 +16,10 @@ use aws_sdk_s3::{ /// A newly created S3 client. #[must_use] pub async fn create_new_client( - endpoint: &str, region_id: &str, access_key_id: &str, secret_access_key: &str, + endpoint: Option<&str>, ) -> Client { let credential = Credentials::new( access_key_id, @@ -30,11 +30,11 @@ pub async fn create_new_client( ); let region = Region::new(region_id.to_owned()); let base_config = aws_config::defaults(BehaviorVersion::latest()).load().await; - let config = Builder::from(&base_config) - .endpoint_url(endpoint) + let mut config_builder = Builder::from(&base_config) .region(region) .credentials_provider(credential) - .force_path_style(true) - .build(); + .force_path_style(true); + config_builder.set_endpoint_url(endpoint.map(std::borrow::ToOwned::to_owned)); + let config = config_builder.build(); Client::from_conf(config) } diff --git a/components/clp-rust-utils/src/sqs/client.rs b/components/clp-rust-utils/src/sqs/client.rs index 4972a568c6..e640b809e4 100644 --- a/components/clp-rust-utils/src/sqs/client.rs +++ b/components/clp-rust-utils/src/sqs/client.rs @@ -12,10 +12,10 @@ use aws_sdk_sqs::{ /// A newly created SQS client. #[must_use] pub async fn create_new_client( - endpoint: &str, region_id: &str, access_key_id: &str, secret_access_key: &str, + endpoint: Option<&str>, ) -> Client { let credential = Credentials::new( access_key_id, @@ -26,10 +26,9 @@ pub async fn create_new_client( ); let region = Region::new(region_id.to_owned()); let base_config = aws_config::defaults(BehaviorVersion::latest()).load().await; - let config = Builder::from(&base_config) - .endpoint_url(endpoint) + let mut config_builder = Builder::from(&base_config) .credentials_provider(credential) - .region(region) - .build(); - Client::from_conf(config) + .region(region); + config_builder.set_endpoint_url(endpoint.map(std::borrow::ToOwned::to_owned)); + Client::from_conf(config_builder.build()) } diff --git a/components/log-ingestor/src/aws_client_manager.rs b/components/log-ingestor/src/aws_client_manager.rs index 6d5baaed5f..a016a66a32 100644 --- a/components/log-ingestor/src/aws_client_manager.rs +++ b/components/log-ingestor/src/aws_client_manager.rs @@ -49,14 +49,9 @@ impl SqsClientWrapper { } pub async fn create(region: &str, access_key_id: &str, secret_access_key: &str) -> Self { - let sqs_endpoint = format!("https://sqs.{region}.amazonaws.com"); - let sqs_client = clp_rust_utils::sqs::create_new_client( - sqs_endpoint.as_str(), - region, - access_key_id, - secret_access_key, - ) - .await; + let sqs_client = + clp_rust_utils::sqs::create_new_client(region, access_key_id, secret_access_key, None) + .await; Self::from(sqs_client) } } @@ -80,14 +75,9 @@ impl S3ClientWrapper { } pub async fn create(region: &str, access_key_id: &str, secret_access_key: &str) -> Self { - let s3_endpoint = format!("https://s3.{region}.amazonaws.com"); - let s3_client = clp_rust_utils::s3::create_new_client( - s3_endpoint.as_str(), - region, - access_key_id, - secret_access_key, - ) - .await; + let s3_client = + clp_rust_utils::s3::create_new_client(region, access_key_id, secret_access_key, None) + .await; Self::from(s3_client) } } diff --git a/components/log-ingestor/tests/test_ingestion_job.rs b/components/log-ingestor/tests/test_ingestion_job.rs index 24d9cc5f04..fc3a3b8275 100644 --- a/components/log-ingestor/tests/test_ingestion_job.rs +++ b/components/log-ingestor/tests/test_ingestion_job.rs @@ -154,10 +154,10 @@ async fn test_sqs_listener() -> Result<()> { let aws_config = AwsConfig::from_env()?; let sqs_client = clp_rust_utils::sqs::create_new_client( - aws_config.endpoint.as_str(), aws_config.region.as_str(), aws_config.access_key_id.as_str(), aws_config.secret_access_key.as_str(), + Some(aws_config.endpoint.as_str()), ) .await; @@ -192,10 +192,10 @@ async fn test_sqs_listener() -> Result<()> { ); let s3_client = clp_rust_utils::s3::create_new_client( - aws_config.endpoint.as_str(), aws_config.region.as_str(), aws_config.access_key_id.as_str(), aws_config.secret_access_key.as_str(), + Some(aws_config.endpoint.as_str()), ) .await; @@ -238,10 +238,10 @@ async fn test_s3_scanner() -> Result<()> { let aws_config = AwsConfig::from_env()?; let s3_client = clp_rust_utils::s3::create_new_client( - aws_config.endpoint.as_str(), aws_config.region.as_str(), aws_config.access_key_id.as_str(), aws_config.secret_access_key.as_str(), + Some(aws_config.endpoint.as_str()), ) .await; From 0d4fb24362ebf8b7808be4050f980a43c0d8050f Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Mon, 8 Dec 2025 11:29:07 -0500 Subject: [PATCH 17/91] refactor(webui): Restructure directory hierarchy to better separate Presto and native components. (#1724) --- .../{Presto => }/ProgressBar/index.module.css | 0 .../{Presto => }/ProgressBar/index.tsx | 6 +++--- .../{ => Native}/NativeControls.tsx | 10 +++++----- .../{ => Native}/QueryInput/index.tsx | 8 ++++---- .../{ => Native}/SearchButton/CancelButton.tsx | 2 +- .../SearchButton/SubmitButton/index.module.css | 0 .../SearchButton/SubmitButton/index.tsx | 12 ++++++------ .../{ => Native}/SearchButton/index.tsx | 4 ++-- .../{ => Native}/search-requests.ts | 8 ++++---- .../SearchControls/{ => Native}/utils.ts | 0 .../pages/SearchPage/SearchControls/index.tsx | 2 +- .../{ => Native}/Message/LogViewerLink.tsx | 4 ++-- .../{ => Native}/Message/index.module.css | 0 .../{ => Native}/Message/index.tsx | 2 +- .../{ => Native}/Message/utils.ts | 0 .../SearchResultsVirtualTable/index.tsx | 4 ++-- .../SearchResultsVirtualTable/typings.tsx | 4 ++-- .../useSearchResults.ts | 8 ++++---- .../SearchResultsTable/{ => Native}/utils.ts | 2 +- .../SearchResults/SearchResultsTable/index.tsx | 2 +- .../{ => Native}/NativeResultsTimeline.tsx | 16 ++++++++-------- .../{ => Native}/useAggregationResults.ts | 8 ++++---- .../SearchResultsTimeline/index.tsx | 2 +- .../webui/client/src/pages/SearchPage/index.tsx | 2 +- 24 files changed, 53 insertions(+), 53 deletions(-) rename components/webui/client/src/pages/SearchPage/{Presto => }/ProgressBar/index.module.css (100%) rename components/webui/client/src/pages/SearchPage/{Presto => }/ProgressBar/index.tsx (85%) rename components/webui/client/src/pages/SearchPage/SearchControls/{ => Native}/NativeControls.tsx (74%) rename components/webui/client/src/pages/SearchPage/SearchControls/{ => Native}/QueryInput/index.tsx (89%) rename components/webui/client/src/pages/SearchPage/SearchControls/{ => Native}/SearchButton/CancelButton.tsx (96%) rename components/webui/client/src/pages/SearchPage/SearchControls/{ => Native}/SearchButton/SubmitButton/index.module.css (100%) rename components/webui/client/src/pages/SearchPage/SearchControls/{ => Native}/SearchButton/SubmitButton/index.tsx (93%) rename components/webui/client/src/pages/SearchPage/SearchControls/{ => Native}/SearchButton/index.tsx (78%) rename components/webui/client/src/pages/SearchPage/SearchControls/{ => Native}/search-requests.ts (94%) rename components/webui/client/src/pages/SearchPage/SearchControls/{ => Native}/utils.ts (100%) rename components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/{ => Native}/Message/LogViewerLink.tsx (93%) rename components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/{ => Native}/Message/index.module.css (100%) rename components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/{ => Native}/Message/index.tsx (96%) rename components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/{ => Native}/Message/utils.ts (100%) rename components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/{ => Native}/SearchResultsVirtualTable/index.tsx (89%) rename components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/{ => Native}/SearchResultsVirtualTable/typings.tsx (93%) rename components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/{ => Native}/SearchResultsVirtualTable/useSearchResults.ts (81%) rename components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/{ => Native}/utils.ts (89%) rename components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/{ => Native}/NativeResultsTimeline.tsx (84%) rename components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/{ => Native}/useAggregationResults.ts (74%) diff --git a/components/webui/client/src/pages/SearchPage/Presto/ProgressBar/index.module.css b/components/webui/client/src/pages/SearchPage/ProgressBar/index.module.css similarity index 100% rename from components/webui/client/src/pages/SearchPage/Presto/ProgressBar/index.module.css rename to components/webui/client/src/pages/SearchPage/ProgressBar/index.module.css diff --git a/components/webui/client/src/pages/SearchPage/Presto/ProgressBar/index.tsx b/components/webui/client/src/pages/SearchPage/ProgressBar/index.tsx similarity index 85% rename from components/webui/client/src/pages/SearchPage/Presto/ProgressBar/index.tsx rename to components/webui/client/src/pages/SearchPage/ProgressBar/index.tsx index 763879cd76..ae6f848e03 100644 --- a/components/webui/client/src/pages/SearchPage/Presto/ProgressBar/index.tsx +++ b/components/webui/client/src/pages/SearchPage/ProgressBar/index.tsx @@ -2,9 +2,9 @@ import {useEffect} from "react"; import {Progress} from "antd"; -import useSearchStore from "../../SearchState"; -import {SEARCH_UI_STATE} from "../../SearchState/typings"; -import {usePseudoProgress} from "../../SearchState/usePseudoProgress"; +import useSearchStore from "../SearchState"; +import {SEARCH_UI_STATE} from "../SearchState/typings"; +import {usePseudoProgress} from "../SearchState/usePseudoProgress"; import styles from "./index.module.css"; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/NativeControls.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/Native/NativeControls.tsx similarity index 74% rename from components/webui/client/src/pages/SearchPage/SearchControls/NativeControls.tsx rename to components/webui/client/src/pages/SearchPage/SearchControls/Native/NativeControls.tsx index 0108dee15c..d7a1116699 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/NativeControls.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/Native/NativeControls.tsx @@ -1,12 +1,12 @@ import {CLP_STORAGE_ENGINES} from "@webui/common/config"; -import {SETTINGS_STORAGE_ENGINE} from "../../../config"; -import Dataset from "./Dataset"; -import styles from "./index.module.css"; +import {SETTINGS_STORAGE_ENGINE} from "../../../../config"; +import Dataset from "../Dataset"; +import styles from "../index.module.css"; +import QueryStatus from "../QueryStatus"; +import TimeRangeInput from "../TimeRangeInput"; import QueryInput from "./QueryInput"; -import QueryStatus from "./QueryStatus"; import SearchButton from "./SearchButton"; -import TimeRangeInput from "./TimeRangeInput"; /** diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/QueryInput/index.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/Native/QueryInput/index.tsx similarity index 89% rename from components/webui/client/src/pages/SearchPage/SearchControls/QueryInput/index.tsx rename to components/webui/client/src/pages/SearchPage/SearchControls/Native/QueryInput/index.tsx index e4b777e102..331c496204 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/QueryInput/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/Native/QueryInput/index.tsx @@ -7,10 +7,10 @@ import { import type {InputRef} from "antd"; -import QueryBox from "../../../../components/QueryBox"; -import useSearchStore from "../../SearchState/index"; -import {SEARCH_UI_STATE} from "../../SearchState/typings"; -import {usePseudoProgress} from "../../SearchState/usePseudoProgress"; +import QueryBox from "../../../../../components/QueryBox"; +import useSearchStore from "../../../SearchState/index"; +import {SEARCH_UI_STATE} from "../../../SearchState/typings"; +import {usePseudoProgress} from "../../../SearchState/usePseudoProgress"; /** diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/CancelButton.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/Native/SearchButton/CancelButton.tsx similarity index 96% rename from components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/CancelButton.tsx rename to components/webui/client/src/pages/SearchPage/SearchControls/Native/SearchButton/CancelButton.tsx index dee14e6cbe..2a21c9c7bf 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/CancelButton.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/Native/SearchButton/CancelButton.tsx @@ -3,7 +3,7 @@ import {useCallback} from "react"; import {CloseOutlined} from "@ant-design/icons"; import {Button} from "antd"; -import useSearchStore from "../../SearchState/index"; +import useSearchStore from "../../../SearchState/index"; import {handleQueryCancel} from "../search-requests"; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton/index.module.css b/components/webui/client/src/pages/SearchPage/SearchControls/Native/SearchButton/SubmitButton/index.module.css similarity index 100% rename from components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton/index.module.css rename to components/webui/client/src/pages/SearchPage/SearchControls/Native/SearchButton/SubmitButton/index.module.css diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton/index.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/Native/SearchButton/SubmitButton/index.tsx similarity index 93% rename from components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton/index.tsx rename to components/webui/client/src/pages/SearchPage/SearchControls/Native/SearchButton/SubmitButton/index.tsx index 7eea0b4bb0..005688d949 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/Native/SearchButton/SubmitButton/index.tsx @@ -9,15 +9,15 @@ import { } from "antd"; import {Dayjs} from "dayjs"; -import {SETTINGS_STORAGE_ENGINE} from "../../../../../config"; -import {computeTimelineConfig} from "../../../SearchResults/SearchResultsTimeline/utils"; -import useSearchStore from "../../../SearchState/index"; -import {SEARCH_UI_STATE} from "../../../SearchState/typings"; -import {handleQuerySubmit} from "../../search-requests"; +import {SETTINGS_STORAGE_ENGINE} from "../../../../../../config"; +import {computeTimelineConfig} from "../../../../SearchResults/SearchResultsTimeline/utils"; +import useSearchStore from "../../../../SearchState/index"; +import {SEARCH_UI_STATE} from "../../../../SearchState/typings"; import { TIME_RANGE_OPTION, TIME_RANGE_OPTION_DAYJS_MAP, -} from "../../TimeRangeInput/utils"; +} from "../../../TimeRangeInput/utils"; +import {handleQuerySubmit} from "../../search-requests"; import styles from "./index.module.css"; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/index.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/Native/SearchButton/index.tsx similarity index 78% rename from components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/index.tsx rename to components/webui/client/src/pages/SearchPage/SearchControls/Native/SearchButton/index.tsx index 2393048c0c..5b18c3830b 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/Native/SearchButton/index.tsx @@ -1,5 +1,5 @@ -import useSearchStore from "../../SearchState/index"; -import {SEARCH_UI_STATE} from "../../SearchState/typings"; +import useSearchStore from "../../../SearchState/index"; +import {SEARCH_UI_STATE} from "../../../SearchState/typings"; import CancelButton from "./CancelButton"; import SubmitButton from "./SubmitButton"; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts b/components/webui/client/src/pages/SearchPage/SearchControls/Native/search-requests.ts similarity index 94% rename from components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts rename to components/webui/client/src/pages/SearchPage/SearchControls/Native/search-requests.ts index a15327c2e5..cde4899cf9 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts +++ b/components/webui/client/src/pages/SearchPage/SearchControls/Native/search-requests.ts @@ -9,10 +9,10 @@ import { cancelQuery, clearQueryResults, submitQuery, -} from "../../../api/search"; -import {SETTINGS_STORAGE_ENGINE} from "../../../config"; -import useSearchStore, {SEARCH_STATE_DEFAULT} from "../SearchState/"; -import {SEARCH_UI_STATE} from "../SearchState/typings"; +} from "../../../../api/search"; +import {SETTINGS_STORAGE_ENGINE} from "../../../../config"; +import useSearchStore, {SEARCH_STATE_DEFAULT} from "../../SearchState"; +import {SEARCH_UI_STATE} from "../../SearchState/typings"; import {unquoteString} from "./utils"; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/utils.ts b/components/webui/client/src/pages/SearchPage/SearchControls/Native/utils.ts similarity index 100% rename from components/webui/client/src/pages/SearchPage/SearchControls/utils.ts rename to components/webui/client/src/pages/SearchPage/SearchControls/Native/utils.ts diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx index 971872c9db..c46186b185 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx @@ -3,7 +3,7 @@ import {CLP_QUERY_ENGINES} from "@webui/common/config"; import {SETTINGS_QUERY_ENGINE} from "../../../config"; import usePrestoSearchState from "../SearchState/Presto"; import {PRESTO_SQL_INTERFACE} from "../SearchState/Presto/typings"; -import NativeControls from "./NativeControls"; +import NativeControls from "./Native/NativeControls"; import FreeformControls from "./Presto/FreeformControls"; import GuidedControls from "./Presto/GuidedControls"; diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/LogViewerLink.tsx b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/Message/LogViewerLink.tsx similarity index 93% rename from components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/LogViewerLink.tsx rename to components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/Message/LogViewerLink.tsx index 206723e60b..78932164d4 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/LogViewerLink.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/Message/LogViewerLink.tsx @@ -7,8 +7,8 @@ import { Typography, } from "antd"; -import {STREAM_TYPE} from "../../../../../config"; -import useSearchStore from "../../../../SearchPage/SearchState"; +import {STREAM_TYPE} from "../../../../../../config"; +import useSearchStore from "../../../../SearchState"; import styles from "./index.module.css"; diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/index.module.css b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/Message/index.module.css similarity index 100% rename from components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/index.module.css rename to components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/Message/index.module.css diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/index.tsx b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/Message/index.tsx similarity index 96% rename from components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/index.tsx rename to components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/Message/index.tsx index e9d8e0ba21..ddb8ee1dfe 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/Message/index.tsx @@ -4,7 +4,7 @@ import {tomorrow} from "react-syntax-highlighter/dist/esm/styles/hljs"; import {CLP_STORAGE_ENGINES} from "@webui/common/config"; import {Typography} from "antd"; -import {SETTINGS_STORAGE_ENGINE} from "../.././../../../config"; +import {SETTINGS_STORAGE_ENGINE} from "../../../../../../config"; import LogViewerLink from "./LogViewerLink"; import {highlighterCustomStyles} from "./utils"; diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/utils.ts b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/Message/utils.ts similarity index 100% rename from components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/utils.ts rename to components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/Message/utils.ts diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/SearchResultsVirtualTable/index.tsx b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/SearchResultsVirtualTable/index.tsx similarity index 89% rename from components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/SearchResultsVirtualTable/index.tsx rename to components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/SearchResultsVirtualTable/index.tsx index ee618d5e29..715c37fec6 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/SearchResultsVirtualTable/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/SearchResultsVirtualTable/index.tsx @@ -1,7 +1,7 @@ import {useEffect} from "react"; -import VirtualTable from "../../../../../components/VirtualTable"; -import useSearchStore from "../../../SearchState/index"; +import VirtualTable from "../../../../../../components/VirtualTable"; +import useSearchStore from "../../../../SearchState/index"; import { SearchResult, searchResultsTableColumns, diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/SearchResultsVirtualTable/typings.tsx b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/SearchResultsVirtualTable/typings.tsx similarity index 93% rename from components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/SearchResultsVirtualTable/typings.tsx rename to components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/SearchResultsVirtualTable/typings.tsx index bb903f1003..d84f4a4a08 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/SearchResultsVirtualTable/typings.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/SearchResultsVirtualTable/typings.tsx @@ -2,8 +2,8 @@ import {CLP_STORAGE_ENGINES} from "@webui/common/config"; import {TableProps} from "antd"; import dayjs from "dayjs"; -import {SETTINGS_STORAGE_ENGINE} from "../../../../../config"; -import {DATETIME_FORMAT_TEMPLATE} from "../../../../../typings/datetime"; +import {SETTINGS_STORAGE_ENGINE} from "../../../../../../config"; +import {DATETIME_FORMAT_TEMPLATE} from "../../../../../../typings/datetime"; import Message from "../Message"; import {getStreamId} from "../utils"; diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/SearchResultsVirtualTable/useSearchResults.ts b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/SearchResultsVirtualTable/useSearchResults.ts similarity index 81% rename from components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/SearchResultsVirtualTable/useSearchResults.ts rename to components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/SearchResultsVirtualTable/useSearchResults.ts index f5b4235607..cfd88155e5 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/SearchResultsVirtualTable/useSearchResults.ts +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/SearchResultsVirtualTable/useSearchResults.ts @@ -1,7 +1,7 @@ -import MongoSocketCollection from "../../../../../api/socket/MongoSocketCollection"; -import {useCursor} from "../../../../../api/socket/useCursor"; -import useSearchStore, {SEARCH_STATE_DEFAULT} from "../../../SearchState/index"; -import {SEARCH_MAX_NUM_RESULTS} from "../typings"; +import MongoSocketCollection from "../../../../../../api/socket/MongoSocketCollection"; +import {useCursor} from "../../../../../../api/socket/useCursor"; +import useSearchStore, {SEARCH_STATE_DEFAULT} from "../../../../SearchState/index"; +import {SEARCH_MAX_NUM_RESULTS} from "../../typings"; import {SearchResult} from "./typings"; diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/utils.ts b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/utils.ts similarity index 89% rename from components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/utils.ts rename to components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/utils.ts index 39af6ef21c..96a8e32659 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/utils.ts +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Native/utils.ts @@ -1,6 +1,6 @@ import {CLP_STORAGE_ENGINES} from "@webui/common/config"; -import {SETTINGS_STORAGE_ENGINE} from "../../../../config"; +import {SETTINGS_STORAGE_ENGINE} from "../../../../../config"; import type {SearchResult} from "./SearchResultsVirtualTable/typings"; diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx index e132d0c4ba..71820701a9 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx @@ -8,8 +8,8 @@ import {CLP_QUERY_ENGINES} from "@webui/common/config"; import {SETTINGS_QUERY_ENGINE} from "../../../../config"; import usePrestoSearchState from "../../SearchState/Presto"; +import SearchResultsVirtualTable from "./Native/SearchResultsVirtualTable"; import PrestoResultsVirtualTable from "./Presto/PrestoResultsVirtualTable"; -import SearchResultsVirtualTable from "./SearchResultsVirtualTable"; import {TABLE_BOTTOM_PADDING} from "./typings"; diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/NativeResultsTimeline.tsx b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/Native/NativeResultsTimeline.tsx similarity index 84% rename from components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/NativeResultsTimeline.tsx rename to components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/Native/NativeResultsTimeline.tsx index 31cc39ac1d..dc2d6ef321 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/NativeResultsTimeline.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/Native/NativeResultsTimeline.tsx @@ -6,15 +6,15 @@ import { import {CLP_STORAGE_ENGINES} from "@webui/common/config"; import {Dayjs} from "dayjs"; -import ResultsTimeline from "../../../../components/ResultsTimeline/index"; -import {TimelineConfig} from "../../../../components/ResultsTimeline/typings"; -import {SETTINGS_STORAGE_ENGINE} from "../../../../config"; -import {handleQuerySubmit} from "../../SearchControls/search-requests"; -import {TIME_RANGE_OPTION} from "../../SearchControls/TimeRangeInput/utils"; -import useSearchStore, {SEARCH_STATE_DEFAULT} from "../../SearchState/index"; -import {SEARCH_UI_STATE} from "../../SearchState/typings"; +import ResultsTimeline from "../../../../../components/ResultsTimeline/index"; +import {TimelineConfig} from "../../../../../components/ResultsTimeline/typings"; +import {SETTINGS_STORAGE_ENGINE} from "../../../../../config"; +import {handleQuerySubmit} from "../../../SearchControls/Native/search-requests"; +import {TIME_RANGE_OPTION} from "../../../SearchControls/TimeRangeInput/utils"; +import useSearchStore, {SEARCH_STATE_DEFAULT} from "../../../SearchState/index"; +import {SEARCH_UI_STATE} from "../../../SearchState/typings"; +import {computeTimelineConfig} from "../utils"; import {useAggregationResults} from "./useAggregationResults"; -import {computeTimelineConfig} from "./utils"; /** diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/useAggregationResults.ts b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/Native/useAggregationResults.ts similarity index 74% rename from components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/useAggregationResults.ts rename to components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/Native/useAggregationResults.ts index 1cacdcbe92..6d6b297892 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/useAggregationResults.ts +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/Native/useAggregationResults.ts @@ -1,7 +1,7 @@ -import MongoSocketCollection from "../../../../api/socket/MongoSocketCollection"; -import {useCursor} from "../../../../api/socket/useCursor"; -import {TimelineBucket} from "../../../../components/ResultsTimeline/typings"; -import useSearchStore, {SEARCH_STATE_DEFAULT} from "../../SearchState/index"; +import MongoSocketCollection from "../../../../../api/socket/MongoSocketCollection"; +import {useCursor} from "../../../../../api/socket/useCursor"; +import {TimelineBucket} from "../../../../../components/ResultsTimeline/typings"; +import useSearchStore, {SEARCH_STATE_DEFAULT} from "../../../SearchState/index"; /** diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/index.tsx b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/index.tsx index 73f599c943..2183ae83fc 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/index.tsx @@ -2,7 +2,7 @@ import {CLP_QUERY_ENGINES} from "@webui/common/config"; import {Card} from "antd"; import {SETTINGS_QUERY_ENGINE} from "../../../../config"; -import NativeResultsTimeline from "./NativeResultsTimeline"; +import NativeResultsTimeline from "./Native/NativeResultsTimeline"; import PrestoResultsTimeline from "./Presto/PrestoResultsTimeline"; diff --git a/components/webui/client/src/pages/SearchPage/index.tsx b/components/webui/client/src/pages/SearchPage/index.tsx index 407821ed53..14ff5a00cf 100644 --- a/components/webui/client/src/pages/SearchPage/index.tsx +++ b/components/webui/client/src/pages/SearchPage/index.tsx @@ -2,7 +2,7 @@ import {CLP_QUERY_ENGINES} from "@webui/common/config"; import {SETTINGS_QUERY_ENGINE} from "../../config"; import styles from "./index.module.css"; -import {ProgressBar} from "./Presto/ProgressBar"; +import {ProgressBar} from "./ProgressBar"; import SearchControls from "./SearchControls"; import SearchResultsTable from "./SearchResults/SearchResultsTable"; import SearchResultsTimeline from "./SearchResults/SearchResultsTimeline"; From 846a4c8f2753ab728ac407d968dfc07d8b937456 Mon Sep 17 00:00:00 2001 From: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> Date: Mon, 8 Dec 2025 16:19:45 -0500 Subject: [PATCH 18/91] feat(log-ingestor): Add ingestion job orchestration: (#1740) * Add APIs to start a new S3-scanning or SQS-listener ingestion job. * Add an API to peacefully stop a running ingestion job. * Refine ingestion job config schemas for submission. * Improve internal logging to enhance end-to-end visibility and traceability during job execution. Co-authored-by: hoophalab <200652805+hoophalab@users.noreply.github.com> --- Cargo.lock | 1 + .../src/clp_config/package/config.rs | 3 +- components/clp-rust-utils/src/job_config.rs | 3 +- .../src/job_config/ingestion.rs | 77 ++++++++--- components/log-ingestor/Cargo.toml | 1 + .../compression/compression_job_submitter.rs | 4 +- .../log-ingestor/src/compression/listener.rs | 10 +- .../src/ingestion_job/s3_scanner.rs | 44 +++--- .../src/ingestion_job/sqs_listener.rs | 37 +++-- .../log-ingestor/src/ingestion_job_manager.rs | 72 +++++----- components/log-ingestor/src/lib.rs | 1 + components/log-ingestor/src/main.rs | 22 +-- components/log-ingestor/src/routes.rs | 126 ++++++++++++++++++ .../log-ingestor/tests/test_ingestion_job.rs | 16 +-- 14 files changed, 304 insertions(+), 113 deletions(-) create mode 100644 components/log-ingestor/src/routes.rs diff --git a/Cargo.lock b/Cargo.lock index 911d087c25..82650ece36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2142,6 +2142,7 @@ dependencies = [ "clp-rust-utils", "const_format", "secrecy", + "serde", "serde_json", "serial_test", "sqlx", diff --git a/components/clp-rust-utils/src/clp_config/package/config.rs b/components/clp-rust-utils/src/clp_config/package/config.rs index 5f1a546e6d..05ff27ae85 100644 --- a/components/clp-rust-utils/src/clp_config/package/config.rs +++ b/components/clp-rust-utils/src/clp_config/package/config.rs @@ -197,6 +197,7 @@ impl Default for StreamOutputStorage { pub struct LogIngestor { pub host: String, pub port: u16, + #[serde(rename = "buffer_timeout")] pub buffer_timeout_sec: u64, pub buffer_size_threshold: u64, pub channel_capacity: usize, @@ -206,7 +207,7 @@ impl Default for LogIngestor { fn default() -> Self { Self { host: "localhost".to_owned(), - port: 3333, + port: 3270, buffer_timeout_sec: 300, buffer_size_threshold: 50 * 1024 * 1024, channel_capacity: 10, diff --git a/components/clp-rust-utils/src/job_config.rs b/components/clp-rust-utils/src/job_config.rs index 919b80bdac..289c6850df 100644 --- a/components/clp-rust-utils/src/job_config.rs +++ b/components/clp-rust-utils/src/job_config.rs @@ -2,9 +2,8 @@ mod clp_io_config; pub use clp_io_config::*; mod compression; -mod ingestion; +pub mod ingestion; mod search; pub use compression::*; -pub use ingestion::*; pub use search::*; diff --git a/components/clp-rust-utils/src/job_config/ingestion.rs b/components/clp-rust-utils/src/job_config/ingestion.rs index 435d41590b..19f0034ab2 100644 --- a/components/clp-rust-utils/src/job_config/ingestion.rs +++ b/components/clp-rust-utils/src/job_config/ingestion.rs @@ -1,24 +1,67 @@ -/// Base configuration for ingesting logs from S3. -#[derive(Clone, Debug)] -pub struct S3IngestionBaseConfig { - /// AWS service region. - pub region: String, +pub mod s3 { + use serde::{Deserialize, Serialize}; - /// The S3 bucket to ingest from. - pub bucket_name: String, + /// Base configuration for ingesting logs from S3. + #[derive(Clone, Debug, Serialize, Deserialize)] + pub struct BaseConfig { + /// AWS service region. + pub region: String, - /// The S3 key prefix to ingest from. - pub key_prefix: String, + /// The S3 bucket to ingest from. + pub bucket_name: String, - /// The dataset to ingest into. Using `default` if not specified. - pub dataset: Option, + /// The S3 key prefix to ingest from. + pub key_prefix: String, - /// The optional key for extracting timestamps from object metadata. - pub timestamp_key: Option, + /// The dataset to ingest into. Defaults to `None` (which uses the default dataset). + #[serde(default)] + pub dataset: Option, - /// Whether to treat the ingested objects as unstructured logs. - pub unstructured: bool, + /// The optional key for extracting timestamps from object metadata. Defaults to `None`. + #[serde(default)] + pub timestamp_key: Option, - /// Tags to apply on the compressed archives. - pub tags: Option>, + /// Whether to treat the ingested objects as unstructured logs. Defaults to `false`. + #[serde(default = "default_unstructured")] + pub unstructured: bool, + + /// Tags to apply on the compressed archives. Defaults to `None`. + #[serde(default)] + pub tags: Option>, + } + + /// Configuration for a SQS listener job. + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct SqsListenerConfig { + #[serde(flatten)] + pub base: BaseConfig, + + /// The SQS queue URL to poll for S3 event notifications. The given queue must be dedicated + /// to this ingestion job. + pub queue_url: String, + } + + /// Configuration for a S3 scanner job. + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct S3ScannerConfig { + #[serde(flatten)] + pub base: BaseConfig, + + /// The scan interval in seconds. Defaults to 30 seconds. + #[serde(default = "default_scanning_interval_sec")] + pub scanning_interval_sec: u32, + + /// The key to ingest after. If specified, only objects with keys lexicographically greater + /// than this value will be ingested. Defaults to `None`. + #[serde(default)] + pub start_after: Option, + } + + const fn default_unstructured() -> bool { + false + } + + const fn default_scanning_interval_sec() -> u32 { + 30 + } } diff --git a/components/log-ingestor/Cargo.toml b/components/log-ingestor/Cargo.toml index 2cd8c7c751..bb8e55e059 100644 --- a/components/log-ingestor/Cargo.toml +++ b/components/log-ingestor/Cargo.toml @@ -13,6 +13,7 @@ clap = { version = "4.5.51", features = ["derive"] } clp-rust-utils = { path = "../clp-rust-utils" } const_format = "0.2.35" secrecy = "0.10.3" +serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.145" sqlx = { version = "0.8.6", features = ["runtime-tokio", "mysql"] } tokio = { version = "1.48.0", features = ["macros", "rt-multi-thread", "signal", "time"] } diff --git a/components/log-ingestor/src/compression/compression_job_submitter.rs b/components/log-ingestor/src/compression/compression_job_submitter.rs index bb060162d3..dafadc3edd 100644 --- a/components/log-ingestor/src/compression/compression_job_submitter.rs +++ b/components/log-ingestor/src/compression/compression_job_submitter.rs @@ -12,8 +12,8 @@ use clp_rust_utils::{ CompressionJobStatus, InputConfig, OutputConfig, - S3IngestionBaseConfig, S3InputConfig, + ingestion::s3::BaseConfig, }, s3::ObjectMetadata, }; @@ -59,7 +59,7 @@ impl CompressionJobSubmitter { db_pool: MySqlPool, aws_credentials: AwsCredentials, archive_output_config: &ArchiveOutput, - ingestion_job_config: &S3IngestionBaseConfig, + ingestion_job_config: &BaseConfig, ) -> Self { let s3_input_config = S3InputConfig { s3_config: S3Config { diff --git a/components/log-ingestor/src/compression/listener.rs b/components/log-ingestor/src/compression/listener.rs index 7599e395c0..3444d9dc90 100644 --- a/components/log-ingestor/src/compression/listener.rs +++ b/components/log-ingestor/src/compression/listener.rs @@ -60,6 +60,10 @@ impl ListenerTask { ); } Some(object_metadata) => { + tracing::debug!( + object = ? object_metadata, + "Received new object metadata." + ); self.buffer.add(object_metadata).await?; } } @@ -112,7 +116,11 @@ impl Listener { }; let cancel_token = CancellationToken::new(); let child_cancel_token = cancel_token.clone(); - let handle = tokio::spawn(async move { task.run(child_cancel_token).await }); + let handle = tokio::spawn(async move { + task.run(child_cancel_token).await.inspect_err(|err| { + tracing::error!(error = ? err, "Listener task execution failed."); + }) + }); Self { sender, diff --git a/components/log-ingestor/src/ingestion_job/s3_scanner.rs b/components/log-ingestor/src/ingestion_job/s3_scanner.rs index e91c53746a..b345c6188e 100644 --- a/components/log-ingestor/src/ingestion_job/s3_scanner.rs +++ b/components/log-ingestor/src/ingestion_job/s3_scanner.rs @@ -2,21 +2,13 @@ use std::time::Duration; use anyhow::Result; use aws_sdk_s3::Client; -use clp_rust_utils::{job_config::S3IngestionBaseConfig, s3::ObjectMetadata}; +use clp_rust_utils::{job_config::ingestion::s3::S3ScannerConfig, s3::ObjectMetadata}; use tokio::{select, sync::mpsc}; use tokio_util::sync::CancellationToken; use uuid::Uuid; use crate::aws_client_manager::AwsClientManagerType; -/// Configuration for a S3 scanner job. -#[derive(Debug, Clone)] -pub struct S3ScannerConfig { - pub base: S3IngestionBaseConfig, - pub scanning_interval: Duration, - pub start_after: Option, -} - /// Represents a S3 scanner task that periodically scans a given prefix under the bucket to fetch /// object metadata for newly created objects. /// @@ -25,6 +17,7 @@ pub struct S3ScannerConfig { /// * [`S3ClientManager`]: The type of the AWS S3 client manager. struct Task> { s3_client_manager: S3ClientManager, + scanning_interval: Duration, config: S3ScannerConfig, sender: mpsc::Sender, } @@ -62,9 +55,16 @@ impl> Task { // simplicity. continue; } - self.sleep().await; } } + + // Sleep for the configured interval or until cancellation is requested. + select! { + () = cancel_token.cancelled() => { + return Ok(()); + } + () = self.sleep() => {} + } } } @@ -109,13 +109,13 @@ impl> Task { if key.ends_with('/') { continue; } - self.sender - .send(ObjectMetadata { - bucket: self.config.base.bucket_name.clone(), - key: key.clone(), - size: size.try_into()?, - }) - .await?; + let object_metadata = ObjectMetadata { + bucket: self.config.base.bucket_name.clone(), + key: key.clone(), + size: size.try_into()?, + }; + tracing::info!(object = ? object_metadata, "Scanned new object metadata on S3."); + self.sender.send(object_metadata).await?; self.config.start_after = Some(key); } @@ -124,7 +124,7 @@ impl> Task { /// Sleeps for the configured scanning interval. pub async fn sleep(&self) { - tokio::time::sleep(self.config.scanning_interval).await; + tokio::time::sleep(self.scanning_interval).await; } } @@ -155,14 +155,20 @@ impl S3Scanner { config: S3ScannerConfig, sender: mpsc::Sender, ) -> Self { + let scanning_interval = Duration::from_secs(u64::from(config.scanning_interval_sec)); let task = Task { s3_client_manager, + scanning_interval, config, sender, }; let cancel_token = CancellationToken::new(); let child_cancel_token = cancel_token.clone(); - let handle = tokio::spawn(async move { task.run(child_cancel_token).await }); + let handle = tokio::spawn(async move { + task.run(child_cancel_token).await.inspect_err(|err| { + tracing::error!(error = ? err, "S3 scanner task execution failed."); + }) + }); Self { id, cancel_token, diff --git a/components/log-ingestor/src/ingestion_job/sqs_listener.rs b/components/log-ingestor/src/ingestion_job/sqs_listener.rs index df88ca7ef5..4f7a524215 100644 --- a/components/log-ingestor/src/ingestion_job/sqs_listener.rs +++ b/components/log-ingestor/src/ingestion_job/sqs_listener.rs @@ -1,7 +1,7 @@ use anyhow::Result; use aws_sdk_sqs::{Client, operation::receive_message::ReceiveMessageOutput}; use clp_rust_utils::{ - job_config::S3IngestionBaseConfig, + job_config::ingestion::s3::SqsListenerConfig, s3::ObjectMetadata, sqs::event::{Record, S3}, }; @@ -11,16 +11,6 @@ use uuid::Uuid; use crate::aws_client_manager::AwsClientManagerType; -/// Configuration for a SQS listener job. -#[derive(Debug, Clone)] -pub struct SqsListenerConfig { - pub queue_url: String, - pub base: S3IngestionBaseConfig, - pub max_num_messages_to_fetch: i32, - pub init_polling_backoff_sec: i32, - pub max_polling_backoff_sec: i32, -} - /// Represents a SQS listener task that listens to SQS messages and extracts S3 object metadata. /// /// # Type Parameters @@ -50,7 +40,9 @@ impl> Task { /// [`aws_sdk_sqs::operation::receive_message::builders::ReceiveMessageFluentBuilder::send`]'s /// return values on failure. pub async fn run(self, cancel_token: CancellationToken) -> Result<()> { - let mut polling_backoff_sec = self.config.init_polling_backoff_sec; + const MAX_NUM_MESSAGES_TO_FETCH: i32 = 10; + const MAX_WAIT_TIME_SEC: i32 = 20; + loop { select! { // Cancellation requested. @@ -62,14 +54,9 @@ impl> Task { result = self.sqs_client_manager.get().await? .receive_message() .queue_url(self.config.queue_url.as_str()) - .max_number_of_messages(self.config.max_num_messages_to_fetch) - .wait_time_seconds(polling_backoff_sec).send() => { - polling_backoff_sec = if self.process_sqs_response(result?).await? { - self.config.init_polling_backoff_sec - } else { std::cmp::min( - polling_backoff_sec.saturating_mul(2), - self.config.max_polling_backoff_sec - ) }; + .max_number_of_messages(MAX_NUM_MESSAGES_TO_FETCH) + .wait_time_seconds(MAX_WAIT_TIME_SEC).send() => { + self.process_sqs_response(result?).await?; } } } @@ -113,6 +100,10 @@ impl> Task { for record in event.records { if let Some(object_metadata) = self.extract_object_metadata(record) { + tracing::info!( + object = ? object_metadata, + "Received new object metadata from SQS." + ); self.sender.send(object_metadata).await?; ingested = true; } @@ -197,7 +188,11 @@ impl SqsListener { }; let cancel_token = CancellationToken::new(); let child_cancel_token = cancel_token.clone(); - let handle = tokio::spawn(async move { task.run(child_cancel_token).await }); + let handle = tokio::spawn(async move { + task.run(child_cancel_token).await.inspect_err(|err| { + tracing::error!(error = ? err, "SQS listener task execution failed."); + }) + }); Self { id, cancel_token, diff --git a/components/log-ingestor/src/ingestion_job_manager.rs b/components/log-ingestor/src/ingestion_job_manager.rs index e5d5d5482c..5c641bca7d 100644 --- a/components/log-ingestor/src/ingestion_job_manager.rs +++ b/components/log-ingestor/src/ingestion_job_manager.rs @@ -9,7 +9,7 @@ use clp_rust_utils::{ credentials::Credentials as ClpCredentials, }, }, - job_config::S3IngestionBaseConfig, + job_config::ingestion::s3::{BaseConfig, S3ScannerConfig, SqsListenerConfig}, s3::ObjectMetadata, }; use tokio::sync::{Mutex, mpsc}; @@ -18,7 +18,7 @@ use uuid::Uuid; use crate::{ aws_client_manager::{S3ClientWrapper, SqsClientWrapper}, compression::{Buffer, CompressionJobSubmitter, Listener}, - ingestion_job::{IngestionJob, S3Scanner, S3ScannerConfig, SqsListenerConfig}, + ingestion_job::{IngestionJob, S3Scanner}, }; /// Errors for ingestion job manager operations. @@ -34,18 +34,13 @@ pub enum Error { PrefixConflict(String), } -/// An async-safe manager for creating and managing ingestion jobs. -pub struct IngestionJobManager { - job_table: Arc>>, - buffer_timeout: Duration, - buffer_size_threshold: u64, - channel_capacity: usize, - aws_credentials: AwsCredentials, - archive_output_config: ArchiveOutput, - mysql_pool: sqlx::MySqlPool, +/// An async-safe state for creating and managing ingestion jobs. +#[derive(Clone)] +pub struct IngestionJobManagerState { + inner: Arc, } -impl IngestionJobManager { +impl IngestionJobManagerState { /// Factory function. /// /// Creates a new ingestion job manager from the given CLP configuration and credentials. @@ -82,15 +77,16 @@ impl IngestionJobManager { 10, ) .await?; - Ok(Self { - job_table: Arc::new(Mutex::new(HashMap::new())), + let inner = Arc::new(IngestionJobManager { + job_table: Mutex::new(HashMap::new()), buffer_timeout: Duration::from_secs(clp_config.log_ingestor.buffer_timeout_sec), buffer_size_threshold: clp_config.log_ingestor.buffer_size_threshold, channel_capacity: clp_config.log_ingestor.channel_capacity, aws_credentials, archive_output_config: clp_config.archive_output, mysql_pool, - }) + }); + Ok(Self { inner }) } /// Creates a new S3 scanner ingestion job. @@ -105,14 +101,13 @@ impl IngestionJobManager { /// /// * Forwards [`Self::create_s3_ingestion_job`]'s return values on failure. pub async fn create_s3_scanner_job(&self, config: S3ScannerConfig) -> Result { - let ingestion_job_config = config.base.clone(); let s3_client_manager = S3ClientWrapper::create( config.base.region.as_str(), - self.aws_credentials.access_key_id.as_str(), - self.aws_credentials.secret_access_key.as_str(), + self.inner.aws_credentials.access_key_id.as_str(), + self.inner.aws_credentials.secret_access_key.as_str(), ) .await; - self.create_s3_ingestion_job(ingestion_job_config, move |job_id, sender| { + self.create_s3_ingestion_job(config.base.clone(), move |job_id, sender| { let scanner = S3Scanner::spawn(job_id, s3_client_manager, config, sender); IngestionJob::S3Scanner(scanner) }) @@ -130,12 +125,12 @@ impl IngestionJobManager { /// Returns an error if: /// /// * Forwards [`Self::create_s3_ingestion_job`]'s return values on failure. - pub async fn create_sqs_listener(&self, config: SqsListenerConfig) -> Result { + pub async fn create_sqs_listener_job(&self, config: SqsListenerConfig) -> Result { let ingestion_job_config = config.base.clone(); let sqs_client_manager = SqsClientWrapper::create( config.base.region.as_str(), - self.aws_credentials.access_key_id.as_str(), - self.aws_credentials.secret_access_key.as_str(), + self.inner.aws_credentials.access_key_id.as_str(), + self.inner.aws_credentials.secret_access_key.as_str(), ) .await; self.create_s3_ingestion_job(ingestion_job_config, move |job_id, sender| { @@ -164,14 +159,16 @@ impl IngestionJobManager { /// * Forwards [`IngestionJob::shutdown_and_join`]'s return value on failure. /// * Forwards [`Listener::shutdown_and_join`]'s return value on failure. pub async fn shutdown_and_remove_job(&self, job_id: Uuid) -> Result<(), Error> { - let mut job_table = self.job_table.lock().await; + let mut job_table = self.inner.job_table.lock().await; let job_to_remove = job_table.remove(&job_id); drop(job_table); match job_to_remove { Some(entry) => { entry.ingestion_job.shutdown_and_join().await?; + tracing::debug!("Ingestion job {} shut down.", job_id); entry.listener.shutdown_and_join().await?; + tracing::debug!("Ingestion job {}'s listener shut down.", job_id); Ok(()) } None => Err(Error::JobNotFound(job_id)), @@ -206,12 +203,12 @@ impl IngestionJobManager { /// * [`Error::PrefixConflict`] if the given key prefix conflicts with an existing job's prefix. async fn create_s3_ingestion_job( &self, - ingestion_job_config: S3IngestionBaseConfig, + ingestion_job_config: BaseConfig, create_ingestion_job: JobCreationCallback, ) -> Result where JobCreationCallback: FnOnce(Uuid, mpsc::Sender) -> IngestionJob, { - let mut job_table = self.job_table.lock().await; + let mut job_table = self.inner.job_table.lock().await; for table_entry in job_table.values() { if table_entry.region != ingestion_job_config.region || table_entry.bucket_name != ingestion_job_config.bucket_name @@ -263,21 +260,32 @@ impl IngestionJobManager { /// # Returns /// /// A new listener for receiving object metadata to ingest. - fn create_listener(&self, ingestion_job_config: &S3IngestionBaseConfig) -> Listener { + fn create_listener(&self, ingestion_job_config: &BaseConfig) -> Listener { let submitter = CompressionJobSubmitter::new( - self.mysql_pool.clone(), - self.aws_credentials.clone(), - &self.archive_output_config, + self.inner.mysql_pool.clone(), + self.inner.aws_credentials.clone(), + &self.inner.archive_output_config, ingestion_job_config, ); Listener::spawn( - Buffer::new(submitter, self.buffer_size_threshold), - self.buffer_timeout, - self.channel_capacity, + Buffer::new(submitter, self.inner.buffer_size_threshold), + self.inner.buffer_timeout, + self.inner.channel_capacity, ) } } +/// Internal shared states for managing ingestion jobs. Must be wrapped in an `Arc`. +struct IngestionJobManager { + job_table: Mutex>, + buffer_timeout: Duration, + buffer_size_threshold: u64, + channel_capacity: usize, + aws_credentials: AwsCredentials, + archive_output_config: ArchiveOutput, + mysql_pool: sqlx::MySqlPool, +} + /// Represents an entry in the ingestion job table. struct IngestionJobTableEntry { ingestion_job: IngestionJob, diff --git a/components/log-ingestor/src/lib.rs b/components/log-ingestor/src/lib.rs index c1dc94280a..1e04dbc2b9 100644 --- a/components/log-ingestor/src/lib.rs +++ b/components/log-ingestor/src/lib.rs @@ -2,3 +2,4 @@ pub mod aws_client_manager; pub mod compression; pub mod ingestion_job; pub mod ingestion_job_manager; +pub mod routes; diff --git a/components/log-ingestor/src/main.rs b/components/log-ingestor/src/main.rs index a58311c544..e6c682b89e 100644 --- a/components/log-ingestor/src/main.rs +++ b/components/log-ingestor/src/main.rs @@ -1,19 +1,22 @@ use anyhow::Context; -use axum::{Router, routing::get}; use clap::Parser; use clp_rust_utils::{clp_config::package, serde::yaml}; +use log_ingestor::{ingestion_job_manager::IngestionJobManagerState, routes::create_router}; use tracing_appender::rolling::{RollingFileAppender, Rotation}; use tracing_subscriber::{self, fmt::writer::MakeWriterExt}; #[derive(Parser)] #[command(version, about = "log-ingestor for CLP.")] struct Args { + /// Path to the CLP config file. #[arg(long)] config: String, + /// Host to bind the server to. #[arg(long)] host: String, + /// Port to bind the server to. #[arg(long)] port: u16, } @@ -78,7 +81,7 @@ async fn shutdown_signal() { async fn main() -> anyhow::Result<()> { let args = Args::parse(); - let (_config, _credentials) = read_config_and_credentials(&args)?; + let (config, credentials) = read_config_and_credentials(&args)?; let _guard = set_up_logging()?; let addr = format!("{}:{}", args.host, args.port); @@ -86,17 +89,16 @@ async fn main() -> anyhow::Result<()> { .await .context(format!("Cannot listen to {addr}"))?; - let app = Router::new() - .route("/", get(health)) - .route("/health", get(health)); + let log_ingestor_manager_state = IngestionJobManagerState::from_config(config, credentials) + .await + .inspect_err(|err| { + tracing::error!(err = ? err, "Failed to create ingestion job manager from CLP config."); + })?; + let log_ingestor_router = create_router().with_state(log_ingestor_manager_state); tracing::info!("Server started at {addr}"); - axum::serve(listener, app) + axum::serve(listener, log_ingestor_router) .with_graceful_shutdown(shutdown_signal()) .await?; Ok(()) } - -async fn health() -> &'static str { - "log-ingestor is running" -} diff --git a/components/log-ingestor/src/routes.rs b/components/log-ingestor/src/routes.rs new file mode 100644 index 0000000000..b999a0e42f --- /dev/null +++ b/components/log-ingestor/src/routes.rs @@ -0,0 +1,126 @@ +use std::str::FromStr; + +use axum::{ + Json, + Router, + extract::{Path, State}, + response::IntoResponse, + routing::{delete, get, post}, +}; +use clp_rust_utils::job_config::ingestion::s3::{S3ScannerConfig, SqsListenerConfig}; +use serde::Serialize; + +use crate::ingestion_job_manager::{Error as IngestionJobManagerError, IngestionJobManagerState}; + +/// Factory method to create an Axum router configured with all log ingestor routes. +/// +/// # Returns +/// +/// A newly created router instance configured without setting the state. +pub fn create_router() -> Router { + Router::new() + .route("/", get(health)) + .route("/health", get(health)) + .route("/s3_scanner", post(create_s3_scanner_job)) + .route("/sqs_listener", post(create_sqs_listener_job)) + .route("/job/{job_id}", delete(stop_and_delete_job)) +} + +#[derive(thiserror::Error, Debug)] +enum Error { + #[error("{0}")] + IngestionJobManagerError(#[from] IngestionJobManagerError), + + #[error("Invalid job ID: {0}")] + InvalidJobId(String), +} + +impl IntoResponse for Error { + fn into_response(self) -> axum::response::Response { + let (status_code, error_message) = match &self { + Self::IngestionJobManagerError(e) => match e { + IngestionJobManagerError::InternalError(_) => ( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + "Internal server error".to_string(), + ), + IngestionJobManagerError::JobNotFound(_) => { + (axum::http::StatusCode::NOT_FOUND, self.to_string()) + } + IngestionJobManagerError::PrefixConflict(_) => { + (axum::http::StatusCode::CONFLICT, self.to_string()) + } + }, + Self::InvalidJobId(_) => (axum::http::StatusCode::BAD_REQUEST, self.to_string()), + }; + let body = serde_json::json!({ + "error": error_message + }); + (status_code, Json(body)).into_response() + } +} + +#[derive(Clone, Serialize)] +struct CreationResponse { + /// The unique ID of the created ingestion job. + id: String, +} + +async fn health() -> &'static str { + "log-ingestor is running" +} + +async fn create_s3_scanner_job( + State(ingestion_job_manager_state): State, + Json(config): Json, +) -> Result, Error> { + tracing::info!(config = ? config, "Create S3 scanner ingestion job."); + let job_id = ingestion_job_manager_state + .create_s3_scanner_job(config) + .await + .map_err(|err| { + tracing::error!(err = ? err, "Failed to create S3 scanner ingestion job."); + Error::IngestionJobManagerError(err) + })?; + tracing::info!(job_id = ? job_id, "Created S3 scanner ingestion job."); + Ok(Json(CreationResponse { + id: job_id.to_string(), + })) +} + +async fn create_sqs_listener_job( + State(ingestion_job_manager_state): State, + Json(config): Json, +) -> Result, Error> { + tracing::info!(config = ? config, "Create SQS listener ingestion job."); + let job_id = ingestion_job_manager_state + .create_sqs_listener_job(config) + .await + .map_err(|err| { + tracing::error!(err = ? err, "Failed to create SQS listener ingestion job."); + Error::IngestionJobManagerError(err) + })?; + tracing::info!(job_id = ? job_id, "Created SQS listener ingestion job."); + Ok(Json(CreationResponse { + id: job_id.to_string(), + })) +} + +async fn stop_and_delete_job( + State(ingestion_job_manager_state): State, + Path(job_id): Path, +) -> Result<(), Error> { + tracing::info!(job_id = ? job_id, "Stop and delete ingestion job."); + let job_id = uuid::Uuid::from_str(&job_id).map_err(|err| { + tracing::error!(err = ? err, "Invalid job ID format."); + Error::InvalidJobId(err.to_string()) + })?; + ingestion_job_manager_state + .shutdown_and_remove_job(job_id) + .await + .map_err(|err| { + tracing::error!(err = ? err, "Failed to stop and delete ingestion job."); + Error::IngestionJobManagerError(err) + })?; + tracing::info!(job_id = ? job_id, "The ingestion job has been deleted."); + Ok(()) +} diff --git a/components/log-ingestor/tests/test_ingestion_job.rs b/components/log-ingestor/tests/test_ingestion_job.rs index fc3a3b8275..6568ae0dfa 100644 --- a/components/log-ingestor/tests/test_ingestion_job.rs +++ b/components/log-ingestor/tests/test_ingestion_job.rs @@ -4,10 +4,13 @@ use std::time::Duration; use anyhow::{Context, Result}; use aws_config::AwsConfig; -use clp_rust_utils::{job_config::S3IngestionBaseConfig, s3::ObjectMetadata}; +use clp_rust_utils::{ + job_config::ingestion::s3::{BaseConfig, S3ScannerConfig, SqsListenerConfig}, + s3::ObjectMetadata, +}; use log_ingestor::{ aws_client_manager::{S3ClientWrapper, SqsClientWrapper}, - ingestion_job::{S3ScannerConfig, SqsListener, SqsListenerConfig}, + ingestion_job::SqsListener, }; use tokio::sync::mpsc; use uuid::Uuid; @@ -168,7 +171,7 @@ async fn test_sqs_listener() -> Result<()> { aws_config.account_id.as_str(), aws_config.queue_name.as_str() ), - base: S3IngestionBaseConfig { + base: BaseConfig { region: aws_config.region.clone(), bucket_name: aws_config.bucket_name.clone(), key_prefix: prefix.clone(), @@ -177,9 +180,6 @@ async fn test_sqs_listener() -> Result<()> { unstructured: false, tags: None, }, - max_num_messages_to_fetch: 2, - init_polling_backoff_sec: 1, - max_polling_backoff_sec: 1, }; let (sender, receiver) = mpsc::channel::(TEST_CHANNEL_CAPACITY); @@ -246,7 +246,7 @@ async fn test_s3_scanner() -> Result<()> { .await; let s3_scanner_config = S3ScannerConfig { - base: S3IngestionBaseConfig { + base: BaseConfig { region: aws_config.region.clone(), bucket_name: aws_config.bucket_name.clone(), key_prefix: prefix.clone(), @@ -255,7 +255,7 @@ async fn test_s3_scanner() -> Result<()> { unstructured: false, tags: None, }, - scanning_interval: Duration::from_millis(300), + scanning_interval_sec: 1, start_after: None, }; From 868844d7ec9b17677f9a1653869fb2647897b3cb Mon Sep 17 00:00:00 2001 From: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> Date: Mon, 8 Dec 2025 18:27:09 -0500 Subject: [PATCH 19/91] fix(lint): Ensure linter-suggested fixes are applied before running Rust code formatting (fixes #1737). (#1750) --- taskfiles/lint.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 242ffa05ac..4c29217b94 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -820,8 +820,10 @@ tasks: fix-rust: cmds: - - task: "fix-rust-format" + # `fix-rust-static` must come before `fix-rust-format` to ensure that any fixes made by the + # static analysis tool (Clippy) are also formatted. - task: "fix-rust-static" + - task: "fix-rust-format" check-rust-format: cmds: From eed6bbfd0986fadd2e900217912dd760c1686a13 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 9 Dec 2025 06:09:51 -0500 Subject: [PATCH 20/91] fix(docker): Ignore errors from clp-package dev image removal (fixes #1370). (#1372) --- tools/docker-images/clp-package/build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/docker-images/clp-package/build.sh b/tools/docker-images/clp-package/build.sh index 51f6656703..d3a7170e1c 100755 --- a/tools/docker-images/clp-package/build.sh +++ b/tools/docker-images/clp-package/build.sh @@ -9,10 +9,10 @@ remove_temp_file_and_prev_image() { [[ -z "$prev_image_id" || "$prev_image_id" == "$new_image_id" ]] && return - if docker image inspect "$prev_image_id" >/dev/null 2>&1; then - echo "Removing previous image $prev_image_id." - docker image remove "$prev_image_id" - fi + docker image inspect "$prev_image_id" >/dev/null 2>&1 || return + + echo "Removing previous image $prev_image_id." + docker image remove "$prev_image_id" || true } trap remove_temp_file_and_prev_image EXIT From 78b4c9d2a0dc6ce7f28623696918c3ec57acfab7 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 9 Dec 2025 20:00:31 +0800 Subject: [PATCH 21/91] fix(deps): Update the commit hash of `yscope-dev-utils` submodule init script to y-scope/yscope-dev-utils@6ec4685. (#1752) --- tools/scripts/deps-download/init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/scripts/deps-download/init.sh b/tools/scripts/deps-download/init.sh index dad4406b60..63a7ee39f1 100755 --- a/tools/scripts/deps-download/init.sh +++ b/tools/scripts/deps-download/init.sh @@ -8,7 +8,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" project_root_dir="$script_dir/../../../" download_dep_script="$script_dir/download-dep.py" -readonly YSCOPE_DEV_UTILS_COMMIT_SHA="307e711fe8bdab29a8aa23bb1f1212748df3f144" +readonly YSCOPE_DEV_UTILS_COMMIT_SHA="6ec46856cfda8cbfb65d9386affbe3bc429186cb" python3 "${download_dep_script}" \ "https://github.com/y-scope/yscope-dev-utils/archive/${YSCOPE_DEV_UTILS_COMMIT_SHA}.zip" \ "yscope-dev-utils-${YSCOPE_DEV_UTILS_COMMIT_SHA}" \ From fe0db96f97bf909b1e4984455228cdcf0cbff8e6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 9 Dec 2025 11:44:21 -0500 Subject: [PATCH 22/91] fix(lint): Strengthen Ruff checks for Python scripts; run error-ignoring check before standard check. (#1670) Co-authored-by: Junhao Liao Co-authored-by: Junhao Liao --- components/clp-mcp-server/pyproject.toml | 2 +- components/clp-package-utils/pyproject.toml | 2 +- components/clp-py-utils/pyproject.toml | 2 +- .../scripts/utils/build-and-run-unit-tests.py | 21 +-- components/job-orchestration/pyproject.toml | 2 +- docs/conf/conf.py | 13 +- lint-requirements.txt | 1 - ruff.toml | 7 +- taskfiles/lint.yaml | 18 +-- tools/deployment/presto-clp/scripts/init.py | 144 ++++++++++++------ tools/scripts/deps-download/download-dep.py | 15 +- tools/scripts/find-broken-docs-links.py | 42 ++++- tools/scripts/localstack/get-free-port.py | 1 + tools/scripts/localstack/stop.py | 1 + 14 files changed, 182 insertions(+), 89 deletions(-) mode change 100644 => 100755 components/core/tools/scripts/utils/build-and-run-unit-tests.py mode change 100644 => 100755 tools/deployment/presto-clp/scripts/init.py mode change 100644 => 100755 tools/scripts/deps-download/download-dep.py mode change 100644 => 100755 tools/scripts/find-broken-docs-links.py diff --git a/components/clp-mcp-server/pyproject.toml b/components/clp-mcp-server/pyproject.toml index 849818f343..de3c5e7071 100644 --- a/components/clp-mcp-server/pyproject.toml +++ b/components/clp-mcp-server/pyproject.toml @@ -47,7 +47,7 @@ show_error_context = true show_error_end = true [tool.ruff] -extend = "../../tools/yscope-dev-utils/exports/lint-configs/python/ruff.toml" +extend = "../../ruff.toml" [tool.uv] default-groups = ["clp", "dev"] diff --git a/components/clp-package-utils/pyproject.toml b/components/clp-package-utils/pyproject.toml index 8ab9ea9ba4..8ea41d459a 100644 --- a/components/clp-package-utils/pyproject.toml +++ b/components/clp-package-utils/pyproject.toml @@ -38,7 +38,7 @@ show_error_context = true show_error_end = true [tool.ruff] -extend = "../../tools/yscope-dev-utils/exports/lint-configs/python/ruff.toml" +extend = "../../ruff.toml" [tool.uv] default-groups = ["clp", "dev"] diff --git a/components/clp-py-utils/pyproject.toml b/components/clp-py-utils/pyproject.toml index a2d947230b..3d983fd09f 100644 --- a/components/clp-py-utils/pyproject.toml +++ b/components/clp-py-utils/pyproject.toml @@ -41,7 +41,7 @@ show_error_context = true show_error_end = true [tool.ruff] -extend = "../../tools/yscope-dev-utils/exports/lint-configs/python/ruff.toml" +extend = "../../ruff.toml" [tool.uv] default-groups = ["clp", "dev"] diff --git a/components/core/tools/scripts/utils/build-and-run-unit-tests.py b/components/core/tools/scripts/utils/build-and-run-unit-tests.py old mode 100644 new mode 100755 index 7c4b136171..48d5cad167 --- a/components/core/tools/scripts/utils/build-and-run-unit-tests.py +++ b/components/core/tools/scripts/utils/build-and-run-unit-tests.py @@ -1,9 +1,13 @@ +#!/usr/bin/env python3 +"""Builds the CLP-core's binaries and runs its unit tests.""" + +from __future__ import annotations + import argparse import logging import subprocess import sys from pathlib import Path -from typing import List, Optional # Set up console logging logging_console_handler = logging.StreamHandler() @@ -21,7 +25,7 @@ logger = logging.getLogger(__name__) -def _config_cmake_project(src_dir: Path, build_dir: Path, use_shared_libs: bool): +def _config_cmake_project(src_dir: Path, build_dir: Path, use_shared_libs: bool) -> None: cmd = [ "cmake", "-S", @@ -34,12 +38,11 @@ def _config_cmake_project(src_dir: Path, build_dir: Path, use_shared_libs: bool) subprocess.run(cmd, check=True) -def _build_project(build_dir: Path, num_jobs: Optional[int]): +def _build_project(build_dir: Path, num_jobs: int | None) -> None: """ :param build_dir: :param num_jobs: Max number of jobs to run when building. """ - cmd = [ "cmake", "--build", @@ -51,12 +54,11 @@ def _build_project(build_dir: Path, num_jobs: Optional[int]): subprocess.run(cmd, check=True) -def _run_unit_tests(build_dir: Path, test_spec: Optional[str]): +def _run_unit_tests(build_dir: Path, test_spec: str | None) -> None: """ :param build_dir: :param test_spec: Catch2 test specification. """ - cmd = [ "./unitTest", ] @@ -65,7 +67,8 @@ def _run_unit_tests(build_dir: Path, test_spec: Optional[str]): subprocess.run(cmd, cwd=build_dir, check=True) -def main(argv: List[str]) -> int: +def main(argv: list[str]) -> int: + """Builds the CLP-core's binaries and runs its unit tests.""" args_parser = argparse.ArgumentParser( description="Builds the CLP-core's binaries and runs its unit tests." ) @@ -87,8 +90,8 @@ def main(argv: List[str]) -> int: src_dir: Path = Path(parsed_args.source_dir) build_dir: Path = Path(parsed_args.build_dir) use_shared_libs: bool = parsed_args.use_shared_libs - num_jobs: Optional[int] = parsed_args.num_jobs - test_spec: Optional[str] = parsed_args.test_spec + num_jobs: int | None = parsed_args.num_jobs + test_spec: str | None = parsed_args.test_spec _config_cmake_project(src_dir, build_dir, use_shared_libs) _build_project(build_dir, num_jobs) diff --git a/components/job-orchestration/pyproject.toml b/components/job-orchestration/pyproject.toml index 5e557b0cd1..728e455862 100644 --- a/components/job-orchestration/pyproject.toml +++ b/components/job-orchestration/pyproject.toml @@ -42,7 +42,7 @@ show_error_context = true show_error_end = true [tool.ruff] -extend = "../../tools/yscope-dev-utils/exports/lint-configs/python/ruff.toml" +extend = "../../ruff.toml" [tool.uv] default-groups = ["clp", "dev"] diff --git a/docs/conf/conf.py b/docs/conf/conf.py index 844cb243de..6c98c0feb1 100644 --- a/docs/conf/conf.py +++ b/docs/conf/conf.py @@ -1,4 +1,7 @@ -from datetime import date +# ruff: noqa: INP001 The Sphinx config file is a standalone script rather than an importable module. +"""Sphinx configuration file for CLP documentation.""" + +from datetime import datetime, timezone from sphinx.application import Sphinx @@ -11,7 +14,8 @@ project = "CLP" # NOTE: We don't include a period after "Inc" since the theme adds one already. -copyright = f"2021-{date.today().year} YScope Inc" +# Ignore A001: Shadows built-in name 'copyright' for Sphinx config. +copyright = f"2021-{datetime.now(tz=timezone.utc).year} YScope Inc" # noqa: A001 # -- General configuration ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration @@ -106,6 +110,11 @@ def setup(app: Sphinx) -> None: + """ + Sets up Sphinx app with custom CSS. + + :param app: + """ app.add_css_file("custom.css") app.connect("source-read", _replace_variable_placeholders) diff --git a/lint-requirements.txt b/lint-requirements.txt index 1f6f2d86fc..15d6cdfd0d 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -4,7 +4,6 @@ clang-format>=21.1.2 clang-tidy~=19.1 # Python -black>=24.4.2 ruff>=0.4.4 # YAML diff --git a/ruff.toml b/ruff.toml index 3438bcfcd1..f2a3ddfbf8 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,6 +1 @@ -line-length = 100 -select = ["I"] - -[lint.isort] -case-sensitive = false -order-by-type = false +extend = "tools/yscope-dev-utils/exports/lint-configs/python/ruff.toml" diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 4c29217b94..32d13ef618 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -95,7 +95,6 @@ tasks: cmds: - task: "py" vars: - BLACK_FLAGS: "--check --diff" RUFF_CHECK_FLAGS: "" RUFF_FORMAT_FLAGS: "--diff" @@ -103,7 +102,6 @@ tasks: cmds: - task: "py" vars: - BLACK_FLAGS: "" RUFF_CHECK_FLAGS: "--fix" RUFF_FORMAT_FLAGS: "" @@ -787,9 +785,15 @@ tasks: - "{{.G_COMPONENTS_DIR}}/clp-py-utils" - "{{.G_COMPONENTS_DIR}}/job-orchestration" requires: - vars: ["BLACK_FLAGS", "RUFF_CHECK_FLAGS", "RUFF_FORMAT_FLAGS"] + vars: ["RUFF_CHECK_FLAGS", "RUFF_FORMAT_FLAGS"] deps: ["venv"] cmds: + - for: + var: "UV_PYTHON_PROJECTS_IGNORE_ERRORS" + cmd: |- + uv run --directory "{{.ITEM}}" ruff check {{.RUFF_CHECK_FLAGS}} --exit-zero . + uv run --directory "{{.ITEM}}" ruff format {{.RUFF_FORMAT_FLAGS}} . + uv run --directory "{{.ITEM}}" mypy . || true - for: - "components/core/tools/scripts/utils" - "tools/deployment" @@ -798,20 +802,14 @@ tasks: cmd: |- . "{{.G_LINT_VENV_DIR}}/bin/activate" cd "{{.ITEM}}" - black --color --line-length 100 {{.BLACK_FLAGS}} . ruff check {{.RUFF_CHECK_FLAGS}} . + ruff format {{.RUFF_FORMAT_FLAGS}} . - for: var: "UV_PYTHON_PROJECTS" cmd: |- uv run --directory "{{.ITEM}}" ruff check {{.RUFF_CHECK_FLAGS}} . uv run --directory "{{.ITEM}}" ruff format {{.RUFF_FORMAT_FLAGS}} . uv run --directory "{{.ITEM}}" mypy . - - for: - var: "UV_PYTHON_PROJECTS_IGNORE_ERRORS" - cmd: |- - uv run --directory "{{.ITEM}}" ruff check {{.RUFF_CHECK_FLAGS}} --exit-zero . - uv run --directory "{{.ITEM}}" ruff format {{.RUFF_FORMAT_FLAGS}} . - uv run --directory "{{.ITEM}}" mypy . || true check-rust: cmds: diff --git a/tools/deployment/presto-clp/scripts/init.py b/tools/deployment/presto-clp/scripts/init.py old mode 100644 new mode 100755 index b3e422cf29..ed7eeae207 --- a/tools/deployment/presto-clp/scripts/init.py +++ b/tools/deployment/presto-clp/scripts/init.py @@ -1,8 +1,11 @@ +#!/usr/bin/env python3 +"""Initializes Presto worker configuration based on CLP package settings.""" + import argparse import logging import sys from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any import psutil import yaml @@ -34,7 +37,8 @@ logger = logging.getLogger(__name__) -def main(argv=None) -> int: +def main(argv: list[str] | None = None) -> int: + """Initializes Presto worker configuration based on CLP package settings.""" if argv is None: argv = sys.argv @@ -63,10 +67,10 @@ def main(argv=None) -> int: ) return 1 - with open(clp_config_file_path, "r") as clp_config_file: + with clp_config_file_path.open("r") as clp_config_file: clp_config = yaml.load(clp_config_file, Loader=yaml.CSafeLoader) - env_vars: Dict[str, str] = {} + env_vars: dict[str, str] = {} if not _add_clp_env_vars(clp_config, clp_config_file_path, clp_package_dir, env_vars): return 1 @@ -82,18 +86,17 @@ def main(argv=None) -> int: ): return 1 - with open(output_file, "w") as output_file_handle: - for key, value in env_vars.items(): - output_file_handle.write(f"{key}={value}\n") + with output_file.open("w") as output_file_handle: + output_file_handle.writelines(f"{key}={value}\n" for key, value in env_vars.items()) return 0 def _add_clp_env_vars( - clp_config: Dict[str, Any], + clp_config: dict[str, Any], clp_config_file_path: Path, clp_package_dir: Path, - env_vars: Dict[str, str], + env_vars: dict[str, str], ) -> bool: """ Adds environment variables for CLP config values to `env_vars`. @@ -106,8 +109,28 @@ def _add_clp_env_vars( """ env_vars["PRESTO_COORDINATOR_CLPPROPERTIES_METADATA_TABLE_PREFIX"] = "clp_" + if not _add_db_env_vars(clp_config, env_vars): + return False + + if not _add_archive_env_vars(clp_config, clp_config_file_path, clp_package_dir, env_vars): + return False + + if not _add_credentials_env_vars(clp_package_dir, env_vars): + return False + + return _add_instance_id_env_var(clp_config, clp_package_dir, env_vars) + + +def _add_db_env_vars(clp_config: dict[str, Any], env_vars: dict[str, str]) -> bool: + """ + Adds environment variables for CLP database config values to `env_vars`. + + :param clp_config: + :param env_vars: + :return: Whether the environment variables were successfully added. + """ database_type = _get_config_value(clp_config, "database.type", "mariadb") - if "mariadb" != database_type and "mysql" != database_type: + if database_type not in {"mariadb", "mysql"}: logger.error( "CLP's database.type must be either mariadb or mysql but found '%s'. Presto" " currently only supports reading metadata from a mariadb or mysql database.", @@ -121,6 +144,24 @@ def _add_clp_env_vars( ) env_vars["PRESTO_COORDINATOR_CLPPROPERTIES_METADATA_DATABASE_NAME"] = database_name + return True + + +def _add_archive_env_vars( + clp_config: dict[str, Any], + clp_config_file_path: Path, + clp_package_dir: Path, + env_vars: dict[str, str], +) -> bool: + """ + Adds environment variables for CLP archive storage config values to `env_vars`. + + :param clp_config: + :param clp_config_file_path: Path to the file containing `clp_config`, for logging. + :param clp_package_dir: + :param env_vars: + :return: Whether the environment variables were successfully added. + """ clp_archive_output_storage_type = _get_config_value( clp_config, "archive_output.storage.type", "fs" ) @@ -135,7 +176,8 @@ def _add_clp_env_vars( clp_package_dir, ) ) - elif "s3" == clp_archive_output_storage_type: + return True + if "s3" == clp_archive_output_storage_type: env_vars["CLP_STAGED_ARCHIVES_DIR"] = str( _get_path_clp_config_value( clp_config, @@ -144,23 +186,30 @@ def _add_clp_env_vars( clp_package_dir, ) ) + return _add_clp_s3_env_vars(clp_config, clp_config_file_path, env_vars) - if not _add_clp_s3_env_vars(clp_config, clp_config_file_path, env_vars): - return False - else: - logger.error( - "'%s' for %s is unsupported.", - clp_archive_output_storage_type, - archive_output_storage_key, - ) - return False + logger.error( + "'%s' for %s is unsupported.", + clp_archive_output_storage_type, + archive_output_storage_key, + ) + return False + +def _add_credentials_env_vars(clp_package_dir: Path, env_vars: dict[str, str]) -> bool: + """ + Adds environment variables for CLP database credentials to `env_vars`. + + :param clp_package_dir: + :param env_vars: + :return: Whether the environment variables were successfully added. + """ credentials_file_path = clp_package_dir / "etc" / "credentials.yaml" if not credentials_file_path.exists(): logger.error("'%s' doesn't exist. Did you start CLP?", credentials_file_path) return False - with open(credentials_file_path, "r") as credentials_file: + with credentials_file_path.open("r") as credentials_file: credentials = yaml.load(credentials_file, Loader=yaml.CSafeLoader) try: @@ -174,17 +223,29 @@ def _add_clp_env_vars( return False env_vars["PRESTO_COORDINATOR_CLPPROPERTIES_METADATA_DATABASE_USER"] = database_username env_vars["PRESTO_COORDINATOR_CLPPROPERTIES_METADATA_DATABASE_PASSWORD"] = database_password + return True + +def _add_instance_id_env_var( + clp_config: dict[str, Any], clp_package_dir: Path, env_vars: dict[str, str] +) -> bool: + """ + Adds environment variable for CLP package instance ID to `env_vars`. + + :param clp_config: + :param clp_package_dir: + :param env_vars: + :return: Whether the environment variable was successfully added. + """ instance_id = _get_clp_package_instance_id(clp_config, clp_package_dir) if instance_id is None: return False env_vars["CLP_PACKAGE_NETWORK_NAME"] = f"clp-package-{instance_id}_default" - return True def _add_clp_s3_env_vars( - clp_config: Dict[str, Any], clp_config_file_path: Path, env_vars: Dict[str, str] + clp_config: dict[str, Any], clp_config_file_path: Path, env_vars: dict[str, str] ) -> bool: """ Adds environment variables for CLP S3 config values to `env_vars`. @@ -195,7 +256,7 @@ def _add_clp_s3_env_vars( :return: Whether the environment variables were successfully added. """ try: - s3_config_key = f"archive_output.storage.s3_config" + s3_config_key = "archive_output.storage.s3_config" s3_bucket = _get_required_config_value( clp_config, f"{s3_config_key}.bucket", clp_config_file_path ) @@ -232,7 +293,7 @@ def _add_clp_s3_env_vars( return True -def _add_memory_env_vars(env_vars: Dict[str, str]) -> bool: +def _add_memory_env_vars(env_vars: dict[str, str]) -> bool: """ Adds memory-related environment variables based on Presto guidelines. @@ -258,7 +319,7 @@ def _add_memory_env_vars(env_vars: Dict[str, str]) -> bool: return True -def _add_worker_env_vars(coordinator_common_env_file_path: Path, env_vars: Dict[str, str]) -> bool: +def _add_worker_env_vars(coordinator_common_env_file_path: Path, env_vars: dict[str, str]) -> bool: """ Adds environment variables for worker config values to `env_vars`. @@ -270,14 +331,12 @@ def _add_worker_env_vars(coordinator_common_env_file_path: Path, env_vars: Dict[ try: env_vars["PRESTO_COORDINATOR_CONFIGPROPERTIES_DISCOVERY_URI"] = ( - f'http://{config["PRESTO_COORDINATOR_SERVICENAME"]}' - f':{config["PRESTO_COORDINATOR_HTTPPORT"]}' + f"http://{config['PRESTO_COORDINATOR_SERVICENAME']}" + f":{config['PRESTO_COORDINATOR_HTTPPORT']}" ) except KeyError as e: - logger.error( - "Missing required key %s in '%s'", - e, - coordinator_common_env_file_path, + logger.exception( + "Missing required key in '%s'.", coordinator_common_env_file_path, exc_info=e ) return False @@ -285,7 +344,7 @@ def _add_worker_env_vars(coordinator_common_env_file_path: Path, env_vars: Dict[ def _generate_worker_clp_properties( - worker_config_template_path: Path, env_vars: Dict[str, str] + worker_config_template_path: Path, env_vars: dict[str, str] ) -> bool: """ Generates a clp.properties file for the worker. @@ -311,15 +370,13 @@ def _generate_worker_clp_properties( return False properties.append(f"{property_name}={env_var_value}") - with open(worker_config_template_path / "clp.properties", "w", encoding="utf-8") as f: + with (worker_config_template_path / "clp.properties").open("w", encoding="utf-8") as f: f.write("\n".join(properties) + "\n") return True -def _get_clp_package_instance_id( - clp_config: Dict[str, Any], clp_package_dir: Path -) -> Optional[str]: +def _get_clp_package_instance_id(clp_config: dict[str, Any], clp_package_dir: Path) -> str | None: """ Retrieves the CLP package instance ID from the logs directory. @@ -327,7 +384,6 @@ def _get_clp_package_instance_id( :param clp_package_dir: :return: The instance ID if it could be read, otherwise `None`. """ - logs_directory = _get_path_clp_config_value( clp_config, "logs_directory", Path("var") / "log", clp_package_dir ) @@ -358,7 +414,7 @@ def _get_clp_package_instance_id( def _get_path_clp_config_value( - clp_config: Dict[str, Any], key: str, default_value: Path, clp_package_dir: Path + clp_config: dict[str, Any], key: str, default_value: Path, clp_package_dir: Path ) -> Path: """ Gets the value corresponding to `key` from `clp_config` as a `Path`. @@ -373,11 +429,10 @@ def _get_path_clp_config_value( value_as_path = Path(value) if value_as_path.is_absolute(): return value_as_path - else: - return clp_package_dir / value_as_path + return clp_package_dir / value_as_path -def _get_required_config_value(config: Dict[str, Any], key: str, config_file_path: Path) -> str: +def _get_required_config_value(config: dict[str, Any], key: str, config_file_path: Path) -> str: """ Gets the value corresponding to `key` from `config`. Logs an error on failure. @@ -395,8 +450,8 @@ def _get_required_config_value(config: Dict[str, Any], key: str, config_file_pat def _get_config_value( - config: Dict[str, Any], key: str, default_value: Optional[Any] = None -) -> Optional[Any]: + config: dict[str, Any], key: str, default_value: Any | None = None +) -> Any | None: """ Gets the value corresponding to `key` from `config` if it exists. @@ -405,7 +460,6 @@ def _get_config_value( :param default_value: The value to return if `key` doesn't exist in `config`. :return: The value corresponding to `key` if it exists, otherwise `default_value`. """ - keys = key.split(".") value = config for k in keys: diff --git a/tools/scripts/deps-download/download-dep.py b/tools/scripts/deps-download/download-dep.py old mode 100644 new mode 100755 index 5866461fd1..eafb51b378 --- a/tools/scripts/deps-download/download-dep.py +++ b/tools/scripts/deps-download/download-dep.py @@ -1,4 +1,6 @@ #!/usr/bin/env python3 +"""Downloads dependency artifacts.""" + import argparse import logging import os @@ -21,7 +23,8 @@ logger.addHandler(logging_console_handler) -def main(argv): +def main(argv: list[str]) -> int: + """Main.""" args_parser = argparse.ArgumentParser(description="Download dependency.") args_parser.add_argument("source_url", help="URL of the source file.") args_parser.add_argument("source_name", help="Name of the source file.") @@ -48,7 +51,7 @@ def main(argv): try: subprocess.run(cmd, check=True) except subprocess.CalledProcessError: - logger.exception(f"Failed to update submodule '{dest_dir}'.") + logger.exception("Failed to update submodule '%s'.", dest_dir) return -1 return 0 @@ -60,7 +63,11 @@ def main(argv): file_path = work_dir / filename # Download file - urllib.request.urlretrieve(source_url, file_path) + if parsed_url.scheme not in {"http", "https"}: + logger.error("Unsupported source URL scheme '%s'.", parsed_url.scheme) + return -1 + + urllib.request.urlretrieve(source_url, file_path) # noqa: S310 URL scheme is checked above if extract_source: # NOTE: We need to convert `file_path` to a str since `unpack_archive` only accepts a # path-like object on Python versions >= 3.7 @@ -73,7 +80,7 @@ def main(argv): source_path = work_dir / source_name if not source_path.exists(): - logger.error(f"Source '{source_path}' doesn't exist.") + logger.error("Source '%s' doesn't exist.", source_path) return -1 if extract_source: diff --git a/tools/scripts/find-broken-docs-links.py b/tools/scripts/find-broken-docs-links.py old mode 100644 new mode 100755 index 1a1eb7427f..3def749a35 --- a/tools/scripts/find-broken-docs-links.py +++ b/tools/scripts/find-broken-docs-links.py @@ -1,10 +1,25 @@ +#!/usr/bin/env python3 +"""Finds broken docs links in the docs.""" + +import logging import os import subprocess import sys from pathlib import Path +# Setup logging +# Create logger +logger = logging.getLogger() +logger.setLevel(logging.INFO) +# Setup console logging +logging_console_handler = logging.StreamHandler() +logging_formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s") +logging_console_handler.setFormatter(logging_formatter) +logger.addHandler(logging_console_handler) + -def main(argv): +def main() -> int: + """Main.""" repo_root = _get_repo_root() found_violation = False @@ -35,7 +50,9 @@ def main(argv): def _get_repo_root() -> Path: path_str = subprocess.check_output( - ["git", "rev-parse", "--show-toplevel"], cwd=Path(__file__).parent, text=True + ["/usr/bin/env", "git", "rev-parse", "--show-toplevel"], + cwd=Path(__file__).parent, + text=True, ) return Path(path_str.strip()) @@ -56,6 +73,7 @@ def _check_tracked_files( # NOTE: "-z" ensures the paths won't be quoted (while delimiting them using '\0') for path_str in subprocess.check_output( [ + "/usr/bin/env", "git", "ls-files", "--cached", @@ -74,7 +92,15 @@ def _check_tracked_files( try: for match in subprocess.check_output( - ["grep", "--extended-regexp", "--line-number", "--with-filename", pattern, path], + [ + "/usr/bin/env", + "grep", + "--extended-regexp", + "--line-number", + "--with-filename", + pattern, + path, + ], cwd=repo_root, text=True, ).splitlines(): @@ -86,7 +112,7 @@ def _check_tracked_files( return found_matches -def _parse_and_print_match(match: str, error_msg: str): +def _parse_and_print_match(match: str, error_msg: str) -> None: """ Parses and prints grep matches in a format relevant to the current environment. :param match: The match to parse and print. @@ -95,11 +121,11 @@ def _parse_and_print_match(match: str, error_msg: str): if os.getenv("GITHUB_ACTIONS") == "true": # Print a GitHub Actions error annotation file, line, _ = match.split(":", 2) - print(f"::error file={file},line={line}::{error_msg}") + logger.error("::error file=%s,line=%s::%s", file, line, error_msg) else: - print(error_msg, file=sys.stderr) - print(match, file=sys.stderr) + logger.error(error_msg) + logger.error(match) if "__main__" == __name__: - sys.exit(main(sys.argv)) + sys.exit(main()) diff --git a/tools/scripts/localstack/get-free-port.py b/tools/scripts/localstack/get-free-port.py index afbb3f1060..07b5cb7f33 100755 --- a/tools/scripts/localstack/get-free-port.py +++ b/tools/scripts/localstack/get-free-port.py @@ -3,6 +3,7 @@ # dependencies = [] # /// """Script to get an unused port and print the port number to stdout.""" + import socket import sys diff --git a/tools/scripts/localstack/stop.py b/tools/scripts/localstack/stop.py index 784188c68c..64d51253b8 100755 --- a/tools/scripts/localstack/stop.py +++ b/tools/scripts/localstack/stop.py @@ -3,6 +3,7 @@ # dependencies = [] # /// """Script to stop a running LocalStack Docker container.""" + import argparse import logging import subprocess From f1fadb3cb8baffe980c593aa0f509a408eab9157 Mon Sep 17 00:00:00 2001 From: Quinn Taylor Mitchell Date: Tue, 9 Dec 2025 13:03:20 -0500 Subject: [PATCH 23/91] feat(integration-tests): Validate that the components of the CLP package are running (fixes #1644). (#1659) Co-authored-by: Bingran Hu --- integration-tests/tests/test_package_start.py | 19 ++----- .../tests/utils/asserting_utils.py | 34 ++++++++++++ integration-tests/tests/utils/config.py | 54 +++++++++++++++++-- integration-tests/tests/utils/docker_utils.py | 35 ++++++++++++ integration-tests/tests/utils/utils.py | 13 +++++ 5 files changed, 137 insertions(+), 18 deletions(-) create mode 100644 integration-tests/tests/utils/docker_utils.py diff --git a/integration-tests/tests/test_package_start.py b/integration-tests/tests/test_package_start.py index ace9dd6e79..2b12b6cf8b 100644 --- a/integration-tests/tests/test_package_start.py +++ b/integration-tests/tests/test_package_start.py @@ -1,16 +1,15 @@ """Integration tests verifying that the CLP package can be started and stopped.""" -import logging - import pytest +from tests.utils.asserting_utils import ( + validate_package_running, +) from tests.utils.clp_mode_utils import CLP_MODE_CONFIGS from tests.utils.config import PackageInstance TEST_MODES = CLP_MODE_CONFIGS.keys() -logger = logging.getLogger(__name__) - @pytest.mark.package @pytest.mark.parametrize("fixt_package_config", TEST_MODES, indirect=True) @@ -20,13 +19,5 @@ def test_clp_package(fixt_package_instance: PackageInstance) -> None: :param fixt_package_instance: """ - # TODO: write code that properly validates that the package is running. This is a placeholder. - mode_name = fixt_package_instance.package_config.mode_name - logger.info("The '%s' package has been spun up successfully.", mode_name) - - component_list = fixt_package_instance.package_config.component_list - logger.info( - "The '%s' package runs with the following required components :\n%s", - mode_name, - "\n".join(component_list), - ) + # Ensure that all package components are running. + validate_package_running(fixt_package_instance) diff --git a/integration-tests/tests/utils/asserting_utils.py b/integration-tests/tests/utils/asserting_utils.py index 3658f83ad3..32ca4b6b7a 100644 --- a/integration-tests/tests/utils/asserting_utils.py +++ b/integration-tests/tests/utils/asserting_utils.py @@ -7,6 +7,9 @@ import pytest +from tests.utils.config import PackageInstance +from tests.utils.docker_utils import list_running_services_in_compose_project + logger = logging.getLogger(__name__) @@ -28,3 +31,34 @@ def run_and_assert(cmd: list[str], **kwargs: Any) -> subprocess.CompletedProcess except subprocess.TimeoutExpired as e: pytest.fail(f"Command timed out: {' '.join(cmd)}: {e}") return proc + + +def validate_package_running(package_instance: PackageInstance) -> None: + """ + Validate that the given package instance is running by checking that the set of services running + in the Compose project exactly matches the list of required components. + + :param package_instance: + :raise pytest.fail: if the sets of running services and required components do not match. + """ + # Get list of services currently running in the Compose project. + instance_id = package_instance.clp_instance_id + project_name = f"clp-package-{instance_id}" + running_services = set(list_running_services_in_compose_project(project_name)) + + # Compare with list of required components. + required_components = set(package_instance.package_config.component_list) + if required_components == running_services: + return + + fail_msg = "Component mismatch." + + missing_components = required_components - running_services + if missing_components: + fail_msg += f"\nMissing components: {missing_components}." + + unexpected_components = running_services - required_components + if unexpected_components: + fail_msg += f"\nUnexpected services: {unexpected_components}." + + pytest.fail(fail_msg) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 3b2040b2a8..f7375d91f7 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -2,11 +2,15 @@ from __future__ import annotations +import re from dataclasses import dataclass, field, InitVar from pathlib import Path -from typing import TYPE_CHECKING import yaml +from clp_py_utils.clp_config import ( + CLP_DEFAULT_LOG_DIRECTORY_PATH, + ClpConfig, +) from tests.utils.utils import ( unlink, @@ -14,9 +18,6 @@ validate_file_exists, ) -if TYPE_CHECKING: - from clp_py_utils.clp_config import ClpConfig - @dataclass(frozen=True) class ClpCorePathConfig: @@ -74,6 +75,9 @@ class PackagePathConfig: #: Directory to store temporary package config files. temp_config_dir: Path = field(init=False, repr=True) + #: Directory where the CLP package writes logs. + clp_log_dir: Path = field(init=False, repr=True) + def __post_init__(self, test_root_dir: Path) -> None: """Validates init values and initializes attributes.""" # Validate that the CLP package directory exists and contains required directories. @@ -93,8 +97,16 @@ def __post_init__(self, test_root_dir: Path) -> None: validate_dir_exists(test_root_dir) object.__setattr__(self, "temp_config_dir", test_root_dir / "temp_config_files") + # Initialize log directory for the package. + object.__setattr__( + self, + "clp_log_dir", + clp_package_dir / CLP_DEFAULT_LOG_DIRECTORY_PATH, + ) + # Create directories if they do not already exist. self.temp_config_dir.mkdir(parents=True, exist_ok=True) + self.clp_log_dir.mkdir(parents=True, exist_ok=True) @property def start_script_path(self) -> Path: @@ -151,11 +163,45 @@ class PackageInstance: #: The configuration for this package instance. package_config: PackageConfig + #: The instance ID of the running package. + clp_instance_id: str = field(init=False, repr=True) + def __post_init__(self) -> None: """Validates init values and initializes attributes.""" # Validate that the temp config file exists. validate_file_exists(self.package_config.temp_config_file_path) + # Set clp_instance_id from instance-id file. + path_config = self.package_config.path_config + clp_instance_id_file_path = path_config.clp_log_dir / "instance-id" + validate_file_exists(clp_instance_id_file_path) + clp_instance_id = self._get_clp_instance_id(clp_instance_id_file_path) + object.__setattr__(self, "clp_instance_id", clp_instance_id) + + @staticmethod + def _get_clp_instance_id(clp_instance_id_file_path: Path) -> str: + """ + Reads the CLP instance ID from the given file and validates its format. + + :param clp_instance_id_file_path: + :return: The 4-character hexadecimal instance ID. + :raise ValueError: If the file cannot be read or contents are not a 4-character hex string. + """ + try: + contents = clp_instance_id_file_path.read_text(encoding="utf-8").strip() + except OSError as err: + err_msg = f"Cannot read instance-id file '{clp_instance_id_file_path}'" + raise ValueError(err_msg) from err + + if not re.fullmatch(r"[0-9a-fA-F]{4}", contents): + err_msg = ( + f"Invalid instance ID in {clp_instance_id_file_path}: expected a 4-character" + f" hexadecimal string, but read {contents}." + ) + raise ValueError(err_msg) + + return contents + @dataclass(frozen=True) class IntegrationTestPathConfig: diff --git a/integration-tests/tests/utils/docker_utils.py b/integration-tests/tests/utils/docker_utils.py new file mode 100644 index 0000000000..ca611fb9b3 --- /dev/null +++ b/integration-tests/tests/utils/docker_utils.py @@ -0,0 +1,35 @@ +"""Provide utility functions related to the use of Docker during integration tests.""" + +import subprocess + +from tests.utils.utils import get_binary_path + + +def list_running_services_in_compose_project(project_name: str) -> list[str]: + """ + Lists running Docker services that belong to the given Docker Compose project. + + :param project_name: + :return: List of the running services that belong to the specified Docker Compose project. + """ + docker_bin = get_binary_path("docker") + + # fmt: off + compose_ps_cmd = [ + docker_bin, + "compose", + "--project-name", project_name, + "ps", + "--format", "{{.Service}}", + ] + # fmt: on + + compose_ps_proc = subprocess.run(compose_ps_cmd, stdout=subprocess.PIPE, text=True, check=True) + + service_names: list[str] = [] + for line in (compose_ps_proc.stdout or "").splitlines(): + service_name_candidate = line.strip() + if service_name_candidate: + service_names.append(service_name_candidate) + + return service_names diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index 54bbaf9148..8b27e6e1c4 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -8,6 +8,19 @@ from typing import IO +def get_binary_path(name: str) -> str: + """ + :param name: Name of the program binary to locate. + :return: Absolute path to the program binary. + :raise RuntimeError: if the program binary is not found on $PATH. + """ + program_binary = shutil.which(name) + if program_binary is None: + err_msg = f"'{name}' not found in PATH" + raise RuntimeError(err_msg) + return program_binary + + def get_env_var(var_name: str) -> str: """ :param var_name: From 73247a92318061e1ddceef3e040238d24f5cb6b8 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 10 Dec 2025 10:39:15 -0500 Subject: [PATCH 24/91] feat(deployment): Add support for root database credentials in Helm templates. (#1759) --- tools/deployment/package-helm/Chart.yaml | 2 +- .../package-helm/templates/database-secret.yaml | 2 ++ .../package-helm/templates/database-statefulset.yaml | 2 +- .../package-helm/templates/db-table-creator-job.yaml | 10 ++++++++++ tools/deployment/package-helm/values.yaml | 2 ++ 5 files changed, 16 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index 9469a9b469..0a9c85ff60 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "clp" -version: "0.1.2-dev.2" +version: "0.1.2-dev.3" description: "A Helm chart for CLP's (Compressed Log Processor) package deployment" type: "application" appVersion: "0.7.1-dev" diff --git a/tools/deployment/package-helm/templates/database-secret.yaml b/tools/deployment/package-helm/templates/database-secret.yaml index 64235ff4a7..5439e66bd6 100644 --- a/tools/deployment/package-helm/templates/database-secret.yaml +++ b/tools/deployment/package-helm/templates/database-secret.yaml @@ -9,3 +9,5 @@ type: "Opaque" stringData: username: {{ .Values.credentials.database.username | quote }} password: {{ .Values.credentials.database.password | quote }} + root_username: {{ .Values.credentials.database.root_username | quote }} + root_password: {{ .Values.credentials.database.root_password | quote }} diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 39fefe81d2..47801e20f7 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -48,7 +48,7 @@ spec: valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "password" + key: "root_password" ports: - name: "database" containerPort: 3306 diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index a3ece1f23e..bc600db429 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -44,6 +44,16 @@ spec: secretKeyRef: name: {{ include "clp.fullname" . }}-database key: "password" + - name: "CLP_DB_ROOT_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "root_username" + - name: "CLP_DB_ROOT_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "root_password" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 46a06833ab..596ec78c05 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -52,6 +52,8 @@ credentials: database: username: "clp-user" password: "pass" + root_username: "root" + root_password: "root-pass" queue: username: "clp-user" From bd2e23bac1ddd06d29bd6a0dbfc4b91c59de19b8 Mon Sep 17 00:00:00 2001 From: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> Date: Wed, 10 Dec 2025 14:48:21 -0500 Subject: [PATCH 25/91] feat(clp-package): Add log-ingestor config interface and Docker Compose orchestration. (#1741) --- .../clp_package_utils/controller.py | 39 +++++++++++++++++++ .../clp-py-utils/clp_py_utils/clp_config.py | 29 ++++++++++++++ .../src/clp_config/package/config.rs | 19 ++++++--- .../log-ingestor/src/ingestion_job_manager.rs | 14 ++++--- .../src/etc/clp-config.template.json.yaml | 14 +++++++ .../src/etc/clp-config.template.text.yaml | 3 ++ .../design-deployment-orchestration.md | 34 +++++++++------- docs/src/user-docs/guides-multi-host.md | 6 +++ .../package/docker-compose-all.yaml | 36 +++++++++++++++++ .../package/docker-compose-base.yaml | 5 +++ .../package/docker-compose-spider-base.yaml | 5 +++ .../package/docker-compose-spider.yaml | 5 +++ tools/deployment/package/docker-compose.yaml | 5 +++ tools/docker-images/clp-package/Dockerfile | 1 + 14 files changed, 188 insertions(+), 27 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 41408618c3..6e7224c4e0 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -32,6 +32,7 @@ DB_COMPONENT_NAME, DeploymentType, GARBAGE_COLLECTOR_COMPONENT_NAME, + LOG_INGESTOR_COMPONENT_NAME, MCP_SERVER_COMPONENT_NAME, OrchestrationType, QUERY_JOBS_TABLE_NAME, @@ -605,6 +606,43 @@ def _set_up_env_for_api_server(self) -> EnvVarsDict: return env_vars + def _set_up_env_for_log_ingestor(self) -> EnvVarsDict: + """ + Sets up environment variables and directories for the log ingestor component. + + :return: Dictionary of environment variables necessary to launch the component. + """ + component_name = LOG_INGESTOR_COMPONENT_NAME + if self._clp_config.log_ingestor is None: + logger.info("%s is not configured, skipping environment setup...", component_name) + return EnvVarsDict({"CLP_LOG_INGESTOR_ENABLED": "0"}) + if self._clp_config.logs_input.type != StorageType.S3: + logger.info( + "%s is only applicable for S3 logs input type, skipping environment setup...", + component_name, + ) + return EnvVarsDict({"CLP_LOG_INGESTOR_ENABLED": "0"}) + logger.info("Setting up environment for %s...", component_name) + + logs_dir = self._clp_config.logs_directory / component_name + resolved_logs_dir = resolve_host_path_in_container(logs_dir) + resolved_logs_dir.mkdir(parents=True, exist_ok=True) + + env_vars = EnvVarsDict() + + # Connection config + env_vars |= { + "CLP_LOG_INGESTOR_HOST": _get_ip_from_hostname(self._clp_config.log_ingestor.host), + "CLP_LOG_INGESTOR_PORT": str(self._clp_config.log_ingestor.port), + } + + # Logging config + env_vars |= { + "CLP_LOG_INGESTOR_LOGGING_LEVEL": self._clp_config.log_ingestor.logging_level, + } + + return env_vars + def _set_up_env_for_webui(self, container_clp_config: ClpConfig) -> EnvVarsDict: """ Sets up environment variables and settings for the Web UI component. @@ -946,6 +984,7 @@ def set_up_env(self) -> None: env_vars |= self._set_up_env_for_query_worker(num_workers) env_vars |= self._set_up_env_for_reducer(num_workers) env_vars |= self._set_up_env_for_api_server() + env_vars |= self._set_up_env_for_log_ingestor() env_vars |= self._set_up_env_for_webui(container_clp_config) env_vars |= self._set_up_env_for_mcp_server() env_vars |= self._set_up_env_for_garbage_collector() diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index e94449d529..6236a9dadc 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -41,6 +41,7 @@ COMPRESSION_WORKER_COMPONENT_NAME = "compression_worker" QUERY_WORKER_COMPONENT_NAME = "query_worker" API_SERVER_COMPONENT_NAME = "api_server" +LOG_INGESTOR_COMPONENT_NAME = "log_ingestor" WEBUI_COMPONENT_NAME = "webui" MCP_SERVER_COMPONENT_NAME = "mcp_server" GARBAGE_COLLECTOR_COMPONENT_NAME = "garbage_collector" @@ -97,6 +98,15 @@ SerializablePath = Annotated[pathlib.Path, PlainSerializer(serialize_path)] ZstdCompressionLevel = Annotated[int, Field(ge=1, le=19)] +LoggingLevelRust = Literal[ + "ERROR", + "WARN", + "INFO", + "DEBUG", + "TRACE", + "OFF", +] + class DeploymentType(KebabCaseStrEnum): BASE = auto() @@ -758,6 +768,15 @@ class ApiServer(BaseModel): default_max_num_query_results: int = 1000 +class LogIngestor(BaseModel): + host: DomainStr = "localhost" + port: Port = 3002 + buffer_flush_timeout: PositiveInt = 300 # seconds + buffer_flush_threshold: PositiveInt = 256 * 1024 * 1024 # 256 MiB + channel_capacity: PositiveInt = 10 + logging_level: LoggingLevelRust = "INFO" + + class Presto(BaseModel): DEFAULT_PORT: ClassVar[int] = 8080 @@ -802,6 +821,7 @@ class ClpConfig(BaseModel): webui: WebUi = WebUi() garbage_collector: GarbageCollector = GarbageCollector() api_server: ApiServer | None = ApiServer() + log_ingestor: LogIngestor | None = LogIngestor() credentials_file_path: SerializablePath = CLP_DEFAULT_CREDENTIALS_FILE_PATH mcp_server: McpServer | None = None @@ -998,6 +1018,15 @@ def dump_to_primitive_dict(self): return d + @model_validator(mode="after") + def validate_log_ingestor_config(self): + if self.log_ingestor is None: + return self + if self.package.storage_engine != StorageEngine.CLP_S: + msg = f"log-ingestor is only compatible with storage engine `{StorageEngine.CLP_S}`." + raise ValueError(msg) + return self + @model_validator(mode="after") def validate_presto_config(self): query_engine = self.package.query_engine diff --git a/components/clp-rust-utils/src/clp_config/package/config.rs b/components/clp-rust-utils/src/clp_config/package/config.rs index 05ff27ae85..a089276be2 100644 --- a/components/clp-rust-utils/src/clp_config/package/config.rs +++ b/components/clp-rust-utils/src/clp_config/package/config.rs @@ -192,25 +192,32 @@ impl Default for StreamOutputStorage { } } +/// Mirror of `clp_py_utils.clp_config.LogIngestor`. +/// +/// # NOTE +/// +/// * The default values must be kept in sync with the Python definition. #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] #[serde(default)] pub struct LogIngestor { pub host: String, pub port: u16, - #[serde(rename = "buffer_timeout")] - pub buffer_timeout_sec: u64, - pub buffer_size_threshold: u64, + #[serde(rename = "buffer_flush_timeout")] + pub buffer_flush_timeout_sec: u64, + pub buffer_flush_threshold: u64, pub channel_capacity: usize, + pub logging_level: String, } impl Default for LogIngestor { fn default() -> Self { Self { host: "localhost".to_owned(), - port: 3270, - buffer_timeout_sec: 300, - buffer_size_threshold: 50 * 1024 * 1024, + port: 3002, + buffer_flush_timeout_sec: 300, + buffer_flush_threshold: 256 * 1024 * 1024, channel_capacity: 10, + logging_level: "INFO".to_owned(), } } } diff --git a/components/log-ingestor/src/ingestion_job_manager.rs b/components/log-ingestor/src/ingestion_job_manager.rs index 5c641bca7d..9ba586811a 100644 --- a/components/log-ingestor/src/ingestion_job_manager.rs +++ b/components/log-ingestor/src/ingestion_job_manager.rs @@ -79,8 +79,10 @@ impl IngestionJobManagerState { .await?; let inner = Arc::new(IngestionJobManager { job_table: Mutex::new(HashMap::new()), - buffer_timeout: Duration::from_secs(clp_config.log_ingestor.buffer_timeout_sec), - buffer_size_threshold: clp_config.log_ingestor.buffer_size_threshold, + buffer_flush_timeout: Duration::from_secs( + clp_config.log_ingestor.buffer_flush_timeout_sec, + ), + buffer_flush_threshold: clp_config.log_ingestor.buffer_flush_threshold, channel_capacity: clp_config.log_ingestor.channel_capacity, aws_credentials, archive_output_config: clp_config.archive_output, @@ -268,8 +270,8 @@ impl IngestionJobManagerState { ingestion_job_config, ); Listener::spawn( - Buffer::new(submitter, self.inner.buffer_size_threshold), - self.inner.buffer_timeout, + Buffer::new(submitter, self.inner.buffer_flush_threshold), + self.inner.buffer_flush_timeout, self.inner.channel_capacity, ) } @@ -278,8 +280,8 @@ impl IngestionJobManagerState { /// Internal shared states for managing ingestion jobs. Must be wrapped in an `Arc`. struct IngestionJobManager { job_table: Mutex>, - buffer_timeout: Duration, - buffer_size_threshold: u64, + buffer_flush_timeout: Duration, + buffer_flush_threshold: u64, channel_capacity: usize, aws_credentials: AwsCredentials, archive_output_config: ArchiveOutput, diff --git a/components/package-template/src/etc/clp-config.template.json.yaml b/components/package-template/src/etc/clp-config.template.json.yaml index 0784e7556d..1452956b59 100644 --- a/components/package-template/src/etc/clp-config.template.json.yaml +++ b/components/package-template/src/etc/clp-config.template.json.yaml @@ -13,6 +13,20 @@ # initial_backoff_ms: 100 # max_backoff_ms: 5000 +## log-ingestor config. Currently, the config is applicable only if `logs_input.type` is "s3". +#log_ingestor: +# host: "localhost" +# port: 3002 +# # The timeout (in seconds) after which the log buffer is flushed for compression if no new input +# # arrives. +# buffer_flush_timeout: 300 +# # The log buffer size (in bytes) that triggers a flush for compression. +# buffer_flush_threshold: 268435456 # 256 MiB +# # The capacity of the internal channel used for communication between an ingestion job and the +# # log buffer. +# channel_capacity: 10 +# logging_level: "INFO" + ## Location (e.g., directory) containing any logs you wish to compress. Must be reachable by all ## workers. #logs_input: diff --git a/components/package-template/src/etc/clp-config.template.text.yaml b/components/package-template/src/etc/clp-config.template.text.yaml index e8d603cd67..f9ca6a0ba4 100644 --- a/components/package-template/src/etc/clp-config.template.text.yaml +++ b/components/package-template/src/etc/clp-config.template.text.yaml @@ -7,6 +7,9 @@ package: # API server config api_server: null +# log-ingestor is not supported in clp-text +log_ingestor: null + ## Location (e.g., directory) containing any logs you wish to compress. Must be reachable by all ## workers. #logs_input: diff --git a/docs/src/dev-docs/design-deployment-orchestration.md b/docs/src/dev-docs/design-deployment-orchestration.md index 130e0bd096..b7c22534c7 100755 --- a/docs/src/dev-docs/design-deployment-orchestration.md +++ b/docs/src/dev-docs/design-deployment-orchestration.md @@ -48,6 +48,7 @@ graph LR garbage_collector["garbage-collector"] webui["webui"] mcp_server["mcp-server"] + log_ingestor["log-ingestor"] %% One-time jobs db_table_creator["db-table-creator"] @@ -72,6 +73,7 @@ graph LR results_cache_indices_creator -->|completed_successfully| mcp_server db_table_creator -->|completed_successfully| garbage_collector results_cache_indices_creator -->|completed_successfully| garbage_collector + db_table_creator -->|completed_successfully| log_ingestor subgraph Databases database @@ -98,6 +100,7 @@ graph LR subgraph Management & UI api_server + log_ingestor garbage_collector webui end @@ -117,21 +120,22 @@ graph LR :::{table} :align: left -| Service | Description | -|-----------------------|-----------------------------------------------------------------| -| database | Database for archive metadata, compression jobs, and query jobs | -| queue | Task queue for schedulers | -| redis | Task result storage for workers | -| compression_scheduler | Scheduler for compression jobs | -| query_scheduler | Scheduler for search/aggregation jobs | -| results_cache | Storage for the workers to return search results to the UI | -| compression_worker | Worker processes for compression jobs | -| query_worker | Worker processes for search/aggregation jobs | -| reducer | Reducers for performing the final stages of aggregation jobs | -| api_server | API server for submitting queries | -| webui | Web server for the UI | -| mcp_server | MCP server for AI agent to access CLP functionalities | -| garbage_collector | Process to manage data retention | +| Service | Description | +|-----------------------|--------------------------------------------------------------------| +| database | Database for archive metadata, compression jobs, and query jobs | +| queue | Task queue for schedulers | +| redis | Task result storage for workers | +| compression_scheduler | Scheduler for compression jobs | +| query_scheduler | Scheduler for search/aggregation jobs | +| results_cache | Storage for the workers to return search results to the UI | +| compression_worker | Worker processes for compression jobs | +| query_worker | Worker processes for search/aggregation jobs | +| reducer | Reducers for performing the final stages of aggregation jobs | +| api_server | API server for submitting queries | +| webui | Web server for the UI | +| mcp_server | MCP server for AI agent to access CLP functionalities | +| garbage_collector | Process to manage data retention | +| log_ingestor | Server for orchestrating and running continuous log ingestion jobs | ::: diff --git a/docs/src/user-docs/guides-multi-host.md b/docs/src/user-docs/guides-multi-host.md index 6ef651b84d..92664dd179 100755 --- a/docs/src/user-docs/guides-multi-host.md +++ b/docs/src/user-docs/guides-multi-host.md @@ -208,6 +208,12 @@ docker compose \ up api-server \ --no-deps --wait +# Start log-ingestor (optional, only if configured) +docker compose \ + --project-name "clp-package-$(cat var/log/instance-id)" \ + up log-ingestor \ + --no-deps --wait + # Start webui docker compose \ --project-name "clp-package-$(cat var/log/instance-id)" \ diff --git a/tools/deployment/package/docker-compose-all.yaml b/tools/deployment/package/docker-compose-all.yaml index b3c1d542d1..2502cf5f52 100644 --- a/tools/deployment/package/docker-compose-all.yaml +++ b/tools/deployment/package/docker-compose-all.yaml @@ -602,3 +602,39 @@ services: "-f", "http://api_server:3001/health" ] + + log-ingestor: + <<: *service_defaults + hostname: "log_ingestor" + deploy: + # Value must be either 0 or 1. Set to 0 to disable log-ingestor. + replicas: "${CLP_LOG_INGESTOR_ENABLED:-1}" + environment: + CLP_LOGS_DIR: "/var/log/log_ingestor" + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" + RUST_LOG: "${CLP_LOG_INGESTOR_LOGGING_LEVEL:-INFO}" + ports: + - host_ip: "${CLP_LOG_INGESTOR_HOST:-127.0.0.1}" + published: "${CLP_LOG_INGESTOR_PORT:-3002}" + target: 3002 + volumes: + - *volume_clp_config_readonly + - *volume_clp_logs + depends_on: + db-table-creator: + condition: "service_completed_successfully" + command: [ + "/opt/clp/bin/log-ingestor", + "--config", "/etc/clp-config.yaml", + "--host", "0.0.0.0", + "--port", "3002" + ] + healthcheck: + <<: *healthcheck_defaults + test: [ + "CMD", + "curl", + "-f", + "http://log_ingestor:3002/health" + ] diff --git a/tools/deployment/package/docker-compose-base.yaml b/tools/deployment/package/docker-compose-base.yaml index 7f0ee5b788..6a88a37816 100644 --- a/tools/deployment/package/docker-compose-base.yaml +++ b/tools/deployment/package/docker-compose-base.yaml @@ -58,3 +58,8 @@ services: extends: file: "docker-compose-all.yaml" service: "api-server" + + log-ingestor: + extends: + file: "docker-compose-all.yaml" + service: "log-ingestor" diff --git a/tools/deployment/package/docker-compose-spider-base.yaml b/tools/deployment/package/docker-compose-spider-base.yaml index 3223f20d81..c19382f0ea 100644 --- a/tools/deployment/package/docker-compose-spider-base.yaml +++ b/tools/deployment/package/docker-compose-spider-base.yaml @@ -45,3 +45,8 @@ services: extends: file: "docker-compose-all.yaml" service: "garbage-collector" + + log-ingestor: + extends: + file: "docker-compose-all.yaml" + service: "log-ingestor" diff --git a/tools/deployment/package/docker-compose-spider.yaml b/tools/deployment/package/docker-compose-spider.yaml index 93482269e4..20b8133b41 100644 --- a/tools/deployment/package/docker-compose-spider.yaml +++ b/tools/deployment/package/docker-compose-spider.yaml @@ -80,3 +80,8 @@ services: extends: file: "docker-compose-all.yaml" service: "reducer" + + log-ingestor: + extends: + file: "docker-compose-all.yaml" + service: "log-ingestor" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 7e2e0d419d..8cba3a4730 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -78,3 +78,8 @@ services: extends: file: "docker-compose-all.yaml" service: "mcp-server" + + log-ingestor: + extends: + file: "docker-compose-all.yaml" + service: "log-ingestor" diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index c7ed59c1ee..e9e0e49d30 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -42,4 +42,5 @@ COPY --link --chown=${UID} ./build/spider/spider-build/src/spider/spider_worker COPY --link --chown=${UID} ./build/nodejs-22/bin/node bin/node-22 COPY --link --chown=${UID} ./build/python-libs/ lib/python3/site-packages/ COPY --link --chown=${UID} ./build/rust-targets/release/api_server bin/ +COPY --link --chown=${UID} ./build/rust-targets/release/log-ingestor bin/ COPY --link --chown=${UID} ./build/webui/ var/www/webui/ From 6c674da73130d3df46149d0e36228865701c25ef Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 10 Dec 2025 16:16:12 -0500 Subject: [PATCH 26/91] fix(clp-package): Updates decompression script with database name change in `ClpConfig` (fixes #1764). (#1765) --- .../clp_package_utils/scripts/native/decompress.py | 3 ++- .../job_orchestration/executor/compress/compression_task.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py index b4b25a2312..12b39ea54a 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py @@ -14,6 +14,7 @@ CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ClpConfig, ClpDbUserType, + ClpDbNameType, Database, ) from clp_py_utils.clp_metadata_db_utils import get_files_table_name @@ -242,7 +243,7 @@ def handle_extract_file_cmd( "--db-type", clp_db_connection_params["type"], "--db-host", clp_db_connection_params["host"], "--db-port", str(clp_db_connection_params["port"]), - "--db-name", clp_db_connection_params["name"], + "--db-name", clp_db_connection_params["names"][ClpDbNameType.CLP], "--db-table-prefix", clp_db_connection_params["table_prefix"], ] # fmt: on diff --git a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py index 1189d8aff1..d80bdb5678 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py +++ b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py @@ -10,6 +10,7 @@ from clp_py_utils.clp_config import ( CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, + ClpDbNameType, ClpDbUserType, COMPRESSION_JOBS_TABLE_NAME, COMPRESSION_TASKS_TABLE_NAME, @@ -196,7 +197,7 @@ def _get_db_connection_args_for_clp_cmd( "--db-port", str(clp_metadata_db_connection_config["port"]), "--db-name", - clp_metadata_db_connection_config["names"][ClpDbUserType.CLP], + clp_metadata_db_connection_config["names"][ClpDbNameType.CLP], "--db-table-prefix", clp_metadata_db_connection_config["table_prefix"], ] From a01d7f654b0518430be8e9ecf295103d51bcbed7 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 10 Dec 2025 16:33:06 -0500 Subject: [PATCH 27/91] docs(clp-package): Add Spider user guide; Update deployment orchestration dev docs. (#1647) Co-authored-by: Junhao Liao --- .../design-deployment-orchestration.md | 95 ++++++++++++------- docs/src/user-docs/guides-multi-host.md | 18 +++- docs/src/user-docs/guides-using-spider.md | 66 +++++++++++++ docs/src/user-docs/index.md | 1 + 4 files changed, 145 insertions(+), 35 deletions(-) create mode 100644 docs/src/user-docs/guides-using-spider.md diff --git a/docs/src/dev-docs/design-deployment-orchestration.md b/docs/src/dev-docs/design-deployment-orchestration.md index b7c22534c7..80bbe0c675 100755 --- a/docs/src/dev-docs/design-deployment-orchestration.md +++ b/docs/src/dev-docs/design-deployment-orchestration.md @@ -41,7 +41,9 @@ graph LR results_cache["results-cache (MongoDB)"] compression_scheduler["compression-scheduler"] query_scheduler["query-scheduler"] + spider_scheduler["spider-scheduler"] compression_worker["compression-worker"] + spider_compression_worker["spider-compression-worker"] query_worker["query-worker"] reducer["reducer"] api_server["api-server"] @@ -55,31 +57,49 @@ graph LR results_cache_indices_creator["results-cache-indices-creator"] %% Dependencies + %% Link 0-1: Database --> Database initialization jobs database -->|healthy| db_table_creator results_cache -->|healthy| results_cache_indices_creator - db_table_creator -->|completed_successfully| compression_scheduler + linkStyle 0,1 stroke:#ffa500 + + %% Link 2-5: Celery dependencies --> Schedulers queue -->|healthy| compression_scheduler redis -->|healthy| compression_scheduler - db_table_creator -->|completed_successfully| query_scheduler queue -->|healthy| query_scheduler redis -->|healthy| query_scheduler + linkStyle 2,3,4,5 stroke:#ff0000 + + %% Link 6: Schedulers --> Workers query_scheduler -->|healthy| reducer - results_cache_indices_creator -->|completed_successfully| reducer + linkStyle 6 stroke:#800080 + + %% Link 7-15: Database initialization job --> Services db_table_creator -->|completed_successfully| api_server - results_cache_indices_creator -->|completed_successfully| api_server - db_table_creator -->|completed_successfully| webui - results_cache_indices_creator -->|completed_successfully| webui - db_table_creator -->|completed_successfully| mcp_server - results_cache_indices_creator -->|completed_successfully| mcp_server + db_table_creator -->|completed_successfully| compression_scheduler db_table_creator -->|completed_successfully| garbage_collector - results_cache_indices_creator -->|completed_successfully| garbage_collector db_table_creator -->|completed_successfully| log_ingestor + db_table_creator -->|completed_successfully| mcp_server + db_table_creator -->|completed_successfully| query_scheduler + db_table_creator -->|completed_successfully| spider_compression_worker + db_table_creator -->|completed_successfully| spider_scheduler + db_table_creator -->|completed_successfully| webui + linkStyle 7,8,9,10,11,12,13,14,15 stroke:#0000ff + + %% Link 16-20: Results cache initialization job --> Services + results_cache_indices_creator -->|completed_successfully| api_server + results_cache_indices_creator -->|completed_successfully| garbage_collector + results_cache_indices_creator -->|completed_successfully| mcp_server + results_cache_indices_creator -->|completed_successfully| reducer + results_cache_indices_creator -->|completed_successfully| webui + linkStyle 16,17,18,19,20 stroke:#008000 subgraph Databases database - queue - redis results_cache + subgraph celery_dependencies[Celery Dependencies] + queue + redis + end end subgraph Initialization jobs @@ -90,10 +110,12 @@ graph LR subgraph Schedulers compression_scheduler query_scheduler + spider_scheduler end subgraph Workers compression_worker + spider_compression_worker query_worker reducer end @@ -109,6 +131,10 @@ graph LR mcp_server end + %% Subgraph styles + style celery_dependencies fill:#ffffe0 + style spider_compression_worker fill:#008080 + style spider_scheduler fill:#008080 +++ **Figure 1**: Orchestration architecture of the services in the CLP package. @@ -120,22 +146,24 @@ graph LR :::{table} :align: left -| Service | Description | -|-----------------------|--------------------------------------------------------------------| -| database | Database for archive metadata, compression jobs, and query jobs | -| queue | Task queue for schedulers | -| redis | Task result storage for workers | -| compression_scheduler | Scheduler for compression jobs | -| query_scheduler | Scheduler for search/aggregation jobs | -| results_cache | Storage for the workers to return search results to the UI | -| compression_worker | Worker processes for compression jobs | -| query_worker | Worker processes for search/aggregation jobs | -| reducer | Reducers for performing the final stages of aggregation jobs | -| api_server | API server for submitting queries | -| webui | Web server for the UI | -| mcp_server | MCP server for AI agent to access CLP functionalities | -| garbage_collector | Process to manage data retention | -| log_ingestor | Server for orchestrating and running continuous log ingestion jobs | +| Service | Description | +|---------------------------|--------------------------------------------------------------------| +| database | Database for archive metadata, compression jobs, and query jobs | +| queue | Task queue for schedulers | +| redis | Task result storage for workers | +| compression_scheduler | Scheduler for compression jobs | +| query_scheduler | Scheduler for search/aggregation jobs | +| spider_scheduler | Scheduler for Spider distributed task execution framework | +| results_cache | Storage for the workers to return search results to the UI | +| compression_worker | Worker processes for compression jobs using Celery | +| spider_compression_worker | Worker processes for compression jobs using Spider | +| query_worker | Worker processes for search/aggregation jobs using Celery | +| reducer | Reducers for performing the final stages of aggregation jobs | +| api_server | API server for submitting queries | +| webui | Web server for the UI | +| mcp_server | MCP server for AI agent to access CLP functionalities | +| garbage_collector | Process to manage data retention | +| log_ingestor | Server for orchestrating and running continuous log ingestion jobs | ::: @@ -214,12 +242,15 @@ instance ID. ### Deployment Types -CLP supports two deployment types determined by the `package.query_engine` configuration setting. +CLP supports four deployment types determined by the `compression_scheduler.type` and +`package.query_engine` configuration setting. -1. **BASE**: For deployments using [Presto][presto-integration] as the query engine. This deployment - only uses `docker-compose.base.yaml`. -2. **FULL**: For deployments using one of CLP's native query engines. This uses both - `docker-compose.base.yaml` and `docker-compose.yaml`. +| Deployment Type | Compression Scheduler | Query Engine | Docker Compose File | +|-----------------|-----------------------|------------------------------|------------------------------------| +| Base | Celery | [Presto][presto-integration] | `docker-compose-base.yaml` | +| Full | Celery | Native | `docker-compose.yaml` | +| Spider Base | Spider | [Presto][presto-integration] | `docker-compose-spider-base.yaml` | +| Spider Full | Spider | Native | `docker-compose-spider.yaml` | ### Implementation details diff --git a/docs/src/user-docs/guides-multi-host.md b/docs/src/user-docs/guides-multi-host.md index 92664dd179..e9e5ecd436 100755 --- a/docs/src/user-docs/guides-multi-host.md +++ b/docs/src/user-docs/guides-multi-host.md @@ -162,13 +162,13 @@ docker compose \ up db-table-creator \ --no-deps -# Start queue +# Start queue (if using Celery) docker compose \ --project-name "clp-package-$(cat var/log/instance-id)" \ up queue \ --no-deps --wait -# Start redis +# Start redis (if using Celery) docker compose \ --project-name "clp-package-$(cat var/log/instance-id)" \ up redis \ @@ -195,6 +195,12 @@ docker compose \ --project-name "clp-package-$(cat var/log/instance-id)" \ up compression-scheduler \ --no-deps --wait + +# Start Spider scheduler (optional, only if using Spider) +docker compose \ + --project-name "clp-package-$(cat var/log/instance-id)" \ + up spider-scheduler \ + --no-deps --wait # Start query scheduler docker compose \ @@ -236,11 +242,17 @@ docker compose \ # Worker services (can be started on multiple hosts) ################################################################################ -# Start compression worker +# Start compression worker (if using Celery) docker compose \ --project-name "clp-package-$(cat var/log/instance-id)" \ up compression-worker \ --no-deps --wait + +# Start Spider compression worker (optional, only if using Spider) +docker compose \ + --project-name "clp-package-$(cat var/log/instance-id)" \ + up spider-compression-worker \ + --no-deps --wait # Start query worker docker compose \ diff --git a/docs/src/user-docs/guides-using-spider.md b/docs/src/user-docs/guides-using-spider.md new file mode 100644 index 0000000000..94f67699ea --- /dev/null +++ b/docs/src/user-docs/guides-using-spider.md @@ -0,0 +1,66 @@ +# Using Spider with CLP + +[Spider] is a fast and scalable distributed task execution engine that can be used to run tasks. +This guide describes how to set up and use Spider with CLP. + +:::{note} +Spider is under active development, and its integration with CLP may change in the future. +Right now, Spider only supports executing CLP compression tasks. Support for search tasks will be added +later. +::: + +## Requirements + +* [CLP][clp-releases] v0.8.0 or higher +* [Docker] v28 or higher +* [Docker Compose][docker-compose] v2.20.2 or higher + +## Set up + +To use Spider for CLP compression tasks, you need to [set up CLP](#setting-up-clp-with-spider) with +Spider in configuration. + +### Setting up CLP with Spider + +1. Follow the [quick-start](quick-start/index.md) guide to download and extract the CLP package, + but don't start the package just yet. +2. Before starting the package, update the package's config file (`etc/clp-config.yaml`) as follows: + + * Change the `compression_scheduler.type` field to `"spider"`. + + ```yaml + compression_scheduler: + type: "spider" + ``` + + * (Optional) Override `database.names.spider` to avoid name conflicts when using self-provisioned database instances. + + ```yaml + database: + names: + spider: "spider-db" + ``` + + * (Optional) Override the `spider_scheduler` default config to change listening host or avoid port conflicts. + + ```yaml + spider_scheduler: + host: "localhost" + port: 6000 + ``` + +3. (Optional) If you do not intend to use generated credentials, set your own Spider credentials in + `etc/credentials.yaml` before starting the package. + + ```yaml + database: + spider_username: "spider-user" + spider_password: "spider-pass" + ``` + +4. Continue following the [quick-start](./quick-start/index.md#using-clp) guide to start CLP. + +[clp-releases]: https://github.com/y-scope/clp/releases +[docker-compose]: https://docs.docker.com/compose/install/ +[Docker]: https://docs.docker.com/engine/install/ +[Spider]: https://github.com/y-scope/spider diff --git a/docs/src/user-docs/index.md b/docs/src/user-docs/index.md index 98403d4c23..715dcefe55 100644 --- a/docs/src/user-docs/index.md +++ b/docs/src/user-docs/index.md @@ -67,6 +67,7 @@ guides-external-database guides-multi-host guides-retention guides-using-presto +guides-using-spider ::: :::{toctree} From 83000681c4514f347aadfcf05077dc2d1100c343 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 10 Dec 2025 18:25:46 -0500 Subject: [PATCH 28/91] refactor(clp-package-utils): Simplify `load_config_file` function and update its usage across scripts. (#1761) Co-authored-by: sitaowang1998 --- .../clp_package_utils/general.py | 22 ++++++++++++++----- .../scripts/archive_manager.py | 6 +---- .../clp_package_utils/scripts/compress.py | 6 +---- .../scripts/compress_from_s3.py | 6 +---- .../scripts/dataset_manager.py | 6 +---- .../clp_package_utils/scripts/decompress.py | 6 +---- .../scripts/native/archive_manager.py | 4 +--- .../scripts/native/compress.py | 2 +- .../scripts/native/dataset_manager.py | 2 +- .../scripts/native/decompress.py | 2 +- .../scripts/native/search.py | 2 +- .../clp_package_utils/scripts/search.py | 6 +---- .../clp_package_utils/scripts/start_clp.py | 6 +---- .../clp_package_utils/scripts/stop_clp.py | 6 +---- 14 files changed, 29 insertions(+), 53 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index da599bbaaf..f0b1e6498d 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -13,6 +13,7 @@ import yaml from clp_py_utils.clp_config import ( CLP_DEFAULT_CREDENTIALS_FILE_PATH, + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLP_SHARED_CONFIG_FILENAME, ClpConfig, CONTAINER_AWS_CONFIG_DIRECTORY, @@ -424,19 +425,28 @@ def validate_config_key_existence(config, key): return value -def load_config_file( - config_file_path: pathlib.Path, default_config_file_path: pathlib.Path, clp_home: pathlib.Path -): +def load_config_file(config_file_path: pathlib.Path) -> ClpConfig: + """ + Loads and validates a CLP configuration file. + + :param config_file_path: + :return: The loaded and validated ClpConfig object. + :raise ValueError: If the specified config file does not exist, and the requested path is not + the path to the default config file. + """ + clp_home = get_clp_home() + default_config_file_path = clp_home / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH + if config_file_path.exists(): raw_clp_config = read_yaml_config_file(config_file_path) if raw_clp_config is None: clp_config = ClpConfig() else: clp_config = ClpConfig.model_validate(raw_clp_config) + elif config_file_path != default_config_file_path: + err_msg = f"Config file '{config_file_path}' does not exist." + raise ValueError(err_msg) else: - if config_file_path != default_config_file_path: - raise ValueError(f"Config file '{config_file_path}' does not exist.") - clp_config = ClpConfig() clp_config.make_config_paths_absolute(clp_home) diff --git a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py index 6f7a6a4251..93e97246bc 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py @@ -172,11 +172,7 @@ def main(argv: list[str]) -> int: # Validate and load config file try: config_file_path: Path = Path(parsed_args.config) - clp_config: ClpConfig = load_config_file( - resolve_host_path_in_container(config_file_path), - resolve_host_path_in_container(default_config_file_path), - clp_home, - ) + clp_config: ClpConfig = load_config_file(resolve_host_path_in_container(config_file_path)) clp_config.validate_logs_dir(True) # Validate and load necessary credentials diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py index e93d086eb4..50add7a944 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/compress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py @@ -178,11 +178,7 @@ def main(argv): # Validate and load config file try: config_file_path = pathlib.Path(parsed_args.config) - clp_config = load_config_file( - resolve_host_path_in_container(config_file_path), - resolve_host_path_in_container(default_config_file_path), - clp_home, - ) + clp_config = load_config_file(resolve_host_path_in_container(config_file_path)) clp_config.validate_logs_dir(True) # Validate and load necessary credentials diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py b/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py index bc64716374..1f0b5da5d2 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py @@ -225,11 +225,7 @@ def main(argv): try: config_file_path = pathlib.Path(parsed_args.config) - clp_config = load_config_file( - resolve_host_path_in_container(config_file_path), - resolve_host_path_in_container(default_config_file_path), - clp_home, - ) + clp_config = load_config_file(resolve_host_path_in_container(config_file_path)) clp_config.validate_logs_dir(True) validate_and_load_db_credentials_file(clp_config, clp_home, False) diff --git a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py index 2737d937d7..72fb62c9d2 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py @@ -96,11 +96,7 @@ def main(argv: list[str]) -> int: # Validate and load config file try: config_file_path = Path(parsed_args.config) - clp_config = load_config_file( - resolve_host_path_in_container(config_file_path), - resolve_host_path_in_container(default_config_file_path), - clp_home, - ) + clp_config = load_config_file(resolve_host_path_in_container(config_file_path)) clp_config.validate_logs_dir(True) # Validate and load necessary credentials diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py index f0aa6e4657..2adfc6588e 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py @@ -52,11 +52,7 @@ def validate_and_load_config( :return: The config object on success, None otherwise. """ try: - clp_config = load_config_file( - resolve_host_path_in_container(config_file_path), - resolve_host_path_in_container(default_config_file_path), - clp_home, - ) + clp_config = load_config_file(resolve_host_path_in_container(config_file_path)) clp_config.validate_logs_dir(True) # Validate and load necessary credentials diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py index b67de011cf..dffdb176af 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py @@ -189,9 +189,7 @@ def main(argv: list[str]) -> int: # Validate and load config file config_file_path: Path = Path(parsed_args.config) try: - clp_config: ClpConfig = load_config_file( - config_file_path, default_config_file_path, clp_home - ) + clp_config: ClpConfig = load_config_file(config_file_path) clp_config.validate_logs_dir() clp_config.database.load_credentials_from_env() except: diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/compress.py b/components/clp-package-utils/clp_package_utils/scripts/native/compress.py index 5571296363..b4de917e7f 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/compress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/compress.py @@ -356,7 +356,7 @@ def main(argv): # Validate and load config file try: config_file_path = pathlib.Path(parsed_args.config) - clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) + clp_config = load_config_file(config_file_path) clp_config.validate_logs_input_config() clp_config.validate_logs_dir() clp_config.database.load_credentials_from_env() diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py b/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py index 569b762e29..721ead9de8 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py @@ -217,7 +217,7 @@ def main(argv: list[str]) -> int: # Validate and load config file config_file_path = Path(parsed_args.config) try: - clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) + clp_config = load_config_file(config_file_path) clp_config.validate_logs_dir() clp_config.database.load_credentials_from_env() except: diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py index 12b39ea54a..d830a907c9 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py @@ -190,7 +190,7 @@ def validate_and_load_config_file( :return: clp_config on success, None otherwise. """ try: - clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) + clp_config = load_config_file(config_file_path) clp_config.validate_archive_output_config() clp_config.validate_logs_dir() clp_config.database.load_credentials_from_env() diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/search.py b/components/clp-package-utils/clp_package_utils/scripts/native/search.py index 0635a8c563..3a3e9d8cd5 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/search.py @@ -298,7 +298,7 @@ def main(argv): # Validate and load config file try: config_file_path = pathlib.Path(parsed_args.config) - clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) + clp_config = load_config_file(config_file_path) clp_config.validate_logs_dir() clp_config.database.load_credentials_from_env() except: diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py index 5d117f0b3b..f8dcdb3655 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/search.py @@ -93,11 +93,7 @@ def main(argv): # Validate and load config file try: config_file_path = pathlib.Path(parsed_args.config) - clp_config = load_config_file( - resolve_host_path_in_container(config_file_path), - resolve_host_path_in_container(default_config_file_path), - clp_home, - ) + clp_config = load_config_file(resolve_host_path_in_container(config_file_path)) clp_config.validate_logs_dir(True) # Validate and load necessary credentials diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 68c4f99605..e0c4a10017 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -53,11 +53,7 @@ def main(argv): try: # Validate and load config file. config_file_path = pathlib.Path(parsed_args.config) - clp_config = load_config_file( - resolve_host_path_in_container(config_file_path), - resolve_host_path_in_container(default_config_file_path), - clp_home, - ) + clp_config = load_config_file(resolve_host_path_in_container(config_file_path)) validate_and_load_db_credentials_file(clp_config, clp_home, True) if clp_config.queue is not None: diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index b450a73027..680e614c04 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -31,11 +31,7 @@ def main(argv): try: config_file_path = pathlib.Path(parsed_args.config) - clp_config = load_config_file( - resolve_host_path_in_container(config_file_path), - resolve_host_path_in_container(default_config_file_path), - clp_home, - ) + clp_config = load_config_file(resolve_host_path_in_container(config_file_path)) except: logger.exception("Failed to load config.") return -1 From 5b05b0cb897ef7f0399a305ee01c00694fe7a982 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 11 Dec 2025 10:07:24 -0500 Subject: [PATCH 29/91] refactor(clp-package): Migrate arg parsing to click in start_clp.py and stop_clp.py. (#1754) Co-authored-by: sitaowang1998 --- .../clp_package_utils/general.py | 2 +- .../scripts/native/decompress.py | 2 +- .../clp_package_utils/scripts/start_clp.py | 89 +++++++++---------- .../clp_package_utils/scripts/stop_clp.py | 47 +++++----- components/clp-package-utils/pyproject.toml | 1 + components/clp-package-utils/uv.lock | 4 +- components/clp-py-utils/uv.lock | 4 +- components/job-orchestration/uv.lock | 10 ++- integration-tests/uv.lock | 10 ++- 9 files changed, 87 insertions(+), 82 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index f0b1e6498d..297617fe51 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -12,8 +12,8 @@ import yaml from clp_py_utils.clp_config import ( - CLP_DEFAULT_CREDENTIALS_FILE_PATH, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + CLP_DEFAULT_CREDENTIALS_FILE_PATH, CLP_SHARED_CONFIG_FILENAME, ClpConfig, CONTAINER_AWS_CONFIG_DIRECTORY, diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py index d830a907c9..5c05a96cec 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py @@ -13,8 +13,8 @@ CLP_DB_USER_ENV_VAR_NAME, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ClpConfig, - ClpDbUserType, ClpDbNameType, + ClpDbUserType, Database, ) from clp_py_utils.clp_metadata_db_utils import get_files_table_name diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index e0c4a10017..fd361b1a19 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -1,8 +1,11 @@ -import argparse +#!/usr/bin/env python3 +"""Script to start the CLP Package.""" + import logging import pathlib import sys +import click from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH from clp_py_utils.core import resolve_host_path_in_container @@ -17,43 +20,44 @@ validate_retention_config, ) -logger = logging.getLogger(__file__) - - -def main(argv): - clp_home = get_clp_home() - default_config_file_path = clp_home / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH - - args_parser = argparse.ArgumentParser(description="Starts CLP") - args_parser.add_argument( - "--config", - "-c", - default=str(default_config_file_path), - help="CLP package configuration file.", - ) - args_parser.add_argument( - "--verbose", - "-v", - action="store_true", - help="Enable debug logging.", - ) - args_parser.add_argument( - "--setup-only", - action="store_true", - help="Validate configuration and prepare directories without starting services.", - ) +logger = logging.getLogger(__name__) - parsed_args = args_parser.parse_args(argv[1:]) - if parsed_args.verbose: +@click.command() +@click.option( + "--config", + "-c", + type=click.Path(exists=False, path_type=pathlib.Path), + default=lambda: get_clp_home() / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + help="CLP package configuration file.", +) +@click.option( + "--setup-only", + is_flag=True, + help="Validate configuration and prepare directories without starting services.", +) +@click.option( + "--verbose", + "-v", + is_flag=True, + help="Enable debug logging.", +) +def main( + config: pathlib.Path, + setup_only: bool, + verbose: bool, +) -> None: + """Starts the CLP Package.""" + if verbose: logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) try: # Validate and load config file. - config_file_path = pathlib.Path(parsed_args.config) - clp_config = load_config_file(resolve_host_path_in_container(config_file_path)) + resolved_config_path = resolve_host_path_in_container(config) + clp_config = load_config_file(resolved_config_path) + clp_home = get_clp_home() validate_and_load_db_credentials_file(clp_config, clp_home, True) if clp_config.queue is not None: @@ -69,9 +73,9 @@ def main(argv): clp_config.validate_data_dir(True) clp_config.validate_logs_dir(True) clp_config.validate_tmp_dir(True) - except: + except Exception: logger.exception("Failed to load config.") - return -1 + sys.exit(1) try: # Create necessary directories. @@ -84,29 +88,24 @@ def main(argv): resolve_host_path_in_container(clp_config.stream_output.get_directory()).mkdir( parents=True, exist_ok=True ) - except: + except Exception: logger.exception("Failed to create necessary directories.") - return -1 + sys.exit(1) try: instance_id = get_or_create_instance_id(clp_config) controller = DockerComposeController(clp_config, instance_id) controller.set_up_env() - if parsed_args.setup_only: + if setup_only: logger.info( "Completed setup. Services not started because `--setup-only` was specified." ) - return 0 + sys.exit(0) controller.start() - except Exception as ex: - if type(ex) == ValueError: - logger.error(f"Failed to start CLP: {ex}") - else: - logger.exception("Failed to start CLP.") - return -1 - - return 0 + except Exception: + logger.exception("Failed to start CLP.") + sys.exit(1) if "__main__" == __name__: - sys.exit(main(sys.argv)) + main() diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 680e614c04..7a03dfcee4 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -1,8 +1,11 @@ -import argparse +#!/usr/bin/env python3 +"""Script to stop the CLP Package.""" + import logging import pathlib import sys +import click from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH from clp_py_utils.core import resolve_host_path_in_container @@ -12,40 +15,34 @@ load_config_file, ) -logger = logging.getLogger(__file__) - - -def main(argv): - clp_home = get_clp_home() - default_config_file_path = clp_home / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH +logger = logging.getLogger(__name__) - args_parser = argparse.ArgumentParser(description="Stops CLP") - args_parser.add_argument( - "--config", - "-c", - default=str(default_config_file_path), - help="CLP package configuration file.", - ) - - parsed_args = args_parser.parse_args(argv[1:]) +@click.command() +@click.option( + "--config", + "-c", + type=click.Path(exists=False, path_type=pathlib.Path), + default=lambda: get_clp_home() / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + help="CLP package configuration file.", +) +def main(config: pathlib.Path) -> None: + """Stops the CLP Package.""" try: - config_file_path = pathlib.Path(parsed_args.config) - clp_config = load_config_file(resolve_host_path_in_container(config_file_path)) - except: + resolved_config_path = resolve_host_path_in_container(config) + clp_config = load_config_file(resolved_config_path) + except Exception: logger.exception("Failed to load config.") - return -1 + sys.exit(1) try: instance_id = get_or_create_instance_id(clp_config) controller = DockerComposeController(clp_config, instance_id) controller.stop() - except: + except Exception: logger.exception("Failed to stop CLP.") - return -1 - - return 0 + sys.exit(1) if "__main__" == __name__: - sys.exit(main(sys.argv)) + main() diff --git a/components/clp-package-utils/pyproject.toml b/components/clp-package-utils/pyproject.toml index 8ea41d459a..aff5e3fc30 100644 --- a/components/clp-package-utils/pyproject.toml +++ b/components/clp-package-utils/pyproject.toml @@ -7,6 +7,7 @@ readme = "README.md" requires-python = ">=3.10" dependencies = [ "Brotli>=1.1.0", + "click>=8.3.1", "msgpack>=1.1.2", "psutil>=7.1.0", "pydantic>=2.12.3", diff --git a/components/clp-package-utils/uv.lock b/components/clp-package-utils/uv.lock index 5f4b96f854..1e5e70b303 100644 --- a/components/clp-package-utils/uv.lock +++ b/components/clp-package-utils/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" [[package]] @@ -206,6 +206,7 @@ version = "0.7.1.dev0" source = { editable = "." } dependencies = [ { name = "brotli" }, + { name = "click" }, { name = "msgpack" }, { name = "psutil" }, { name = "pydantic" }, @@ -226,6 +227,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "brotli", specifier = ">=1.1.0" }, + { name = "click", specifier = ">=8.3.1" }, { name = "msgpack", specifier = ">=1.1.2" }, { name = "psutil", specifier = ">=7.1.0" }, { name = "pydantic", specifier = ">=2.12.3" }, diff --git a/components/clp-py-utils/uv.lock b/components/clp-py-utils/uv.lock index 827117f8a5..0986538388 100644 --- a/components/clp-py-utils/uv.lock +++ b/components/clp-py-utils/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" [[package]] @@ -206,6 +206,7 @@ version = "0.7.1.dev0" source = { directory = "../clp-package-utils" } dependencies = [ { name = "brotli" }, + { name = "click" }, { name = "msgpack" }, { name = "psutil" }, { name = "pydantic" }, @@ -216,6 +217,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "brotli", specifier = ">=1.1.0" }, + { name = "click", specifier = ">=8.3.1" }, { name = "msgpack", specifier = ">=1.1.2" }, { name = "psutil", specifier = ">=7.1.0" }, { name = "pydantic", specifier = ">=2.12.3" }, diff --git a/components/job-orchestration/uv.lock b/components/job-orchestration/uv.lock index 70c8e7aad7..a83cbb9e30 100644 --- a/components/job-orchestration/uv.lock +++ b/components/job-orchestration/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" [[package]] @@ -153,14 +153,14 @@ redis = [ [[package]] name = "click" -version = "8.3.0" +version = "8.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/46/61/de6cd827efad202d7057d93e0fed9294b96952e188f7384832791c7b2254/click-8.3.0.tar.gz", hash = "sha256:e7b8232224eba16f4ebe410c25ced9f7875cb5f3263ffc93cc3e8da705e229c4", size = 276943, upload-time = "2025-09-18T17:32:23.696Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/db/d3/9dcc0f5797f070ec8edf30fbadfb200e71d9db6b84d211e3b2085a7589a0/click-8.3.0-py3-none-any.whl", hash = "sha256:9b9f285302c6e3064f4330c05f05b81945b2a39544279343e6e7c5f27a9baddc", size = 107295, upload-time = "2025-09-18T17:32:22.42Z" }, + { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, ] [[package]] @@ -206,6 +206,7 @@ version = "0.7.1.dev0" source = { directory = "../clp-package-utils" } dependencies = [ { name = "brotli" }, + { name = "click" }, { name = "msgpack" }, { name = "psutil" }, { name = "pydantic" }, @@ -216,6 +217,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "brotli", specifier = ">=1.1.0" }, + { name = "click", specifier = ">=8.3.1" }, { name = "msgpack", specifier = ">=1.1.2" }, { name = "psutil", specifier = ">=7.1.0" }, { name = "pydantic", specifier = ">=2.12.3" }, diff --git a/integration-tests/uv.lock b/integration-tests/uv.lock index f87258549b..b8f166e609 100644 --- a/integration-tests/uv.lock +++ b/integration-tests/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" [[package]] @@ -420,14 +420,14 @@ wheels = [ [[package]] name = "click" -version = "8.3.0" +version = "8.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/46/61/de6cd827efad202d7057d93e0fed9294b96952e188f7384832791c7b2254/click-8.3.0.tar.gz", hash = "sha256:e7b8232224eba16f4ebe410c25ced9f7875cb5f3263ffc93cc3e8da705e229c4", size = 276943, upload-time = "2025-09-18T17:32:23.696Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/db/d3/9dcc0f5797f070ec8edf30fbadfb200e71d9db6b84d211e3b2085a7589a0/click-8.3.0-py3-none-any.whl", hash = "sha256:9b9f285302c6e3064f4330c05f05b81945b2a39544279343e6e7c5f27a9baddc", size = 107295, upload-time = "2025-09-18T17:32:22.42Z" }, + { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, ] [[package]] @@ -507,6 +507,7 @@ version = "0.7.1.dev0" source = { editable = "../components/clp-package-utils" } dependencies = [ { name = "brotli" }, + { name = "click" }, { name = "msgpack" }, { name = "psutil" }, { name = "pydantic" }, @@ -517,6 +518,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "brotli", specifier = ">=1.1.0" }, + { name = "click", specifier = ">=8.3.1" }, { name = "msgpack", specifier = ">=1.1.2" }, { name = "psutil", specifier = ">=7.1.0" }, { name = "pydantic", specifier = ">=2.12.3" }, From 3f44babfc9f93beca167966e1e942d0650a4523b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 11 Dec 2025 15:49:26 -0500 Subject: [PATCH 30/91] chore: Remove redundant comments from `credentials.template.yaml` (resolves #1743). (#1748) --- components/package-template/src/etc/credentials.template.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/components/package-template/src/etc/credentials.template.yaml b/components/package-template/src/etc/credentials.template.yaml index bbe11721fb..fa1bfd473c 100644 --- a/components/package-template/src/etc/credentials.template.yaml +++ b/components/package-template/src/etc/credentials.template.yaml @@ -1,4 +1,3 @@ -## Database credentials #database: # username: "clp-user" # password: "pass" @@ -7,11 +6,9 @@ # spider_username: "spider-user" # spider_password: "spider-pass" # -## Queue credentials #queue: # username: "clp-user" # password: "pass" # -## Redis credentials #redis: # password: "pass" From 8b5472e64a094fee21fb74b08ccd673650db79f5 Mon Sep 17 00:00:00 2001 From: Quinn Taylor Mitchell Date: Fri, 12 Dec 2025 10:25:25 -0500 Subject: [PATCH 31/91] feat(integration-tests): Add port range assignment based on a configurable base port to avoid package spin-up failures. (#1662) Co-authored-by: Bingran Hu --- integration-tests/tests/conftest.py | 16 ++ .../tests/fixtures/package_config.py | 12 ++ .../tests/fixtures/package_instance.py | 7 +- integration-tests/tests/utils/config.py | 3 + integration-tests/tests/utils/port_utils.py | 166 ++++++++++++++++++ 5 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 integration-tests/tests/utils/port_utils.py diff --git a/integration-tests/tests/conftest.py b/integration-tests/tests/conftest.py index 224fefcd0d..c1af599c98 100644 --- a/integration-tests/tests/conftest.py +++ b/integration-tests/tests/conftest.py @@ -1,5 +1,7 @@ """Global pytest setup.""" +import pytest + # Make the fixtures defined in `tests/fixtures/` globally available without imports. pytest_plugins = [ "tests.fixtures.integration_test_logs", @@ -7,3 +9,17 @@ "tests.fixtures.package_instance", "tests.fixtures.package_config", ] + + +def pytest_addoption(parser: pytest.Parser) -> None: + """ + Adds options for pytest. + + :param parser: + """ + parser.addoption( + "--base-port", + action="store", + default="55000", + help="Base port for CLP package integration tests.", + ) diff --git a/integration-tests/tests/fixtures/package_config.py b/integration-tests/tests/fixtures/package_config.py index 7f40edf9a3..8db533dcf2 100644 --- a/integration-tests/tests/fixtures/package_config.py +++ b/integration-tests/tests/fixtures/package_config.py @@ -9,6 +9,7 @@ get_required_component_list, ) from tests.utils.config import PackageConfig, PackagePathConfig +from tests.utils.port_utils import assign_ports_from_base @pytest.fixture @@ -21,11 +22,21 @@ def fixt_package_config( :param request: :return: An iterator that yields the PackageConfig object for the specified mode. + :raise ValueError: if the CLP base port's value is invalid. """ mode_name: str = request.param clp_config_obj = get_clp_config_from_mode(mode_name) + # Assign ports based on the clp base port CLI option. + base_port_string = request.config.getoption("--base-port") + try: + base_port = int(base_port_string) + except ValueError as err: + err_msg = f"Invalid value '{base_port_string}' for '--base-port'; expected an integer." + raise ValueError(err_msg) from err + assign_ports_from_base(base_port, clp_config_obj) + required_components = get_required_component_list(clp_config_obj) # Construct PackageConfig. @@ -34,6 +45,7 @@ def fixt_package_config( mode_name=mode_name, component_list=required_components, clp_config=clp_config_obj, + base_port=base_port, ) try: diff --git a/integration-tests/tests/fixtures/package_instance.py b/integration-tests/tests/fixtures/package_instance.py index 310e7eaf77..b2ea7ef4b3 100644 --- a/integration-tests/tests/fixtures/package_instance.py +++ b/integration-tests/tests/fixtures/package_instance.py @@ -29,6 +29,11 @@ def fixt_package_instance(fixt_package_config: PackageConfig) -> Iterator[Packag instance = PackageInstance(package_config=fixt_package_config) yield instance except RuntimeError: - pytest.fail(f"Failed to start the {mode_name} package.") + base_port = fixt_package_config.base_port + pytest.fail( + f"Failed to start the {mode_name} package. This could mean that one of the ports" + f" derived from base_port={base_port} was unavailable. You can specify a new value for" + " base_port with the '--base-port' flag." + ) finally: stop_clp_package(fixt_package_config) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index f7375d91f7..2e62e0afd8 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -135,6 +135,9 @@ class PackageConfig: #: The Pydantic representation of a CLP package configuration. clp_config: ClpConfig + #: The base port from which all ports for the components are derived. + base_port: int + def __post_init__(self) -> None: """Write the temporary config file for this package.""" self._write_temp_config_file() diff --git a/integration-tests/tests/utils/port_utils.py b/integration-tests/tests/utils/port_utils.py new file mode 100644 index 0000000000..cfb1b47c45 --- /dev/null +++ b/integration-tests/tests/utils/port_utils.py @@ -0,0 +1,166 @@ +"""Functions for facilitating the port connections for the CLP package.""" + +import socket +from dataclasses import dataclass +from typing import Any + +from clp_py_utils.clp_config import ( + ClpConfig, + REDUCER_COMPONENT_NAME, +) + +# Port constants. +MIN_NON_PRIVILEGED_PORT = 1024 +MAX_PORT = 65535 +VALID_PORT_RANGE = range(MIN_NON_PRIVILEGED_PORT, MAX_PORT + 1) + +# Practical maximum number of reducer instances. +REDUCER_MAX_PORTS = 128 + +# Attribute names used to discover port configuration in component objects. +PORT_ATTR_NAMES = ["port", "base_port"] + + +@dataclass +class PortAssignment: + """ + Represents a port assignment for a CLP component. + + :param component: The component configuration object that needs port assignment. + :param attr_name: The name of the port attribute on the component. + :param port_count: The number of consecutive ports needed by this component. + """ + + component: Any + attr_name: str + port_count: int + + +def assign_ports_from_base(base_port: int, clp_config: ClpConfig) -> None: + """ + Assign ports to all components in `clp_config` that require them, starting from `base_port`. + Ports are assigned sequentially, with each component receiving the number of ports it requires. + + :param base_port: + :param clp_config: + :raise ValueError: If the base port is out of range, or if any required port is in use. + """ + # Discover which components need port assignments. + port_assignments = _discover_port_assignments(clp_config) + total_ports_needed = sum(assignment.port_count for assignment in port_assignments) + + # Validate that all required ports are valid and available. + port_range = range(base_port, base_port + total_ports_needed) + _validate_port_range_bounds(port_range) + _check_ports_available(host="127.0.0.1", port_range=port_range) + + # Assign ports to the components. + current_port = base_port + for assignment in port_assignments: + setattr(assignment.component, assignment.attr_name, current_port) + current_port += assignment.port_count + + +def _check_ports_available(host: str, port_range: range) -> None: + """ + Check that all ports in the given range are available for binding. + + :param host: + :param port_range: + :raise ValueError: If any port in the range is already in use. + """ + for port in port_range: + if not _is_port_free(port=port, host=host): + range_str = _format_port_range(port_range) + err_msg = ( + f"Port '{port}' in the desired range ({range_str}) is already in use. " + "Choose a different port range for the test environment." + ) + raise ValueError(err_msg) + + +def _discover_port_assignments(clp_config: ClpConfig) -> list[PortAssignment]: + """ + Discover which components in `clp_config` require port assignments. + + :param clp_config: + :return: A list of PortAssignment objects describing what ports each component needs. + """ + port_assignments: list[PortAssignment] = [] + + for component_name, component_config in vars(clp_config).items(): + # Skip private attributes and None values. + if component_name.startswith("_") or component_config is None: + continue + + # Check if this component has a port attribute. + port_attr_name = None + for attr_name in PORT_ATTR_NAMES: + if hasattr(component_config, attr_name): + port_attr_name = attr_name + break + if port_attr_name is None: + continue + + # Determine how many ports this component needs. + port_count = REDUCER_MAX_PORTS if component_name == REDUCER_COMPONENT_NAME else 1 + + port_assignments.append( + PortAssignment( + component=component_config, + attr_name=port_attr_name, + port_count=port_count, + ) + ) + + return port_assignments + + +def _format_port_range(port_range: range) -> str: + """ + Format a port range as a human-readable string. + + :param port_range: + :return: A string like "'1024' to '65535' inclusive". + """ + start_port = port_range.start + end_port = port_range.stop - 1 + return f"'{start_port}' to '{end_port}' inclusive" + + +def _is_port_free(port: int, host: str) -> bool: + """ + Check whether a TCP port is available for binding. + + :param port: + :param host: + :return: True if the port can be bound, otherwise False. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + try: + sock.bind((host, port)) + except OSError: + return False + return True + + +def _validate_port_range_bounds(port_range: range) -> None: + """ + Validates that the given port range falls within the valid port range. + + :param port_range: + :raise ValueError: If any part of the range falls outside valid port numbers. + """ + start_port = port_range.start + end_port = port_range.stop - 1 + min_valid_port = VALID_PORT_RANGE.start + max_valid_port = VALID_PORT_RANGE.stop - 1 + + if start_port < min_valid_port or end_port > max_valid_port: + required_range_str = _format_port_range(port_range) + valid_range_str = _format_port_range(VALID_PORT_RANGE) + err_msg = ( + f"The port range derived from --base-port ({required_range_str}) must fall within" + f" the range of valid ports ({valid_range_str})." + ) + raise ValueError(err_msg) From c422366dbaa6faba122b9cea6b860cc6a91b56e4 Mon Sep 17 00:00:00 2001 From: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> Date: Fri, 12 Dec 2025 19:27:50 -0500 Subject: [PATCH 32/91] feat(clp-rust-utils): Use `NonEmptyString` in S3-related configs to ensure validation at the type level. (#1769) --- Cargo.lock | 101 ++++++++++++------ components/clp-rust-utils/Cargo.toml | 1 + .../clp-rust-utils/src/clp_config/package.rs | 10 +- .../src/clp_config/s3_config.rs | 5 +- .../src/job_config/clp_io_config.rs | 9 +- .../src/job_config/ingestion.rs | 15 +-- components/clp-rust-utils/src/lib.rs | 1 + components/clp-rust-utils/src/s3.rs | 5 +- components/clp-rust-utils/src/sqs/event/s3.rs | 5 +- components/clp-rust-utils/src/types.rs | 32 ++++++ .../clp-rust-utils/tests/clp_config_test.rs | 10 +- components/log-ingestor/Cargo.toml | 2 + .../compression/compression_job_submitter.rs | 2 +- .../src/ingestion_job/s3_scanner.rs | 12 ++- .../log-ingestor/src/ingestion_job_manager.rs | 7 +- components/log-ingestor/tests/aws_config.rs | 15 +-- .../tests/test_compression_listener.rs | 7 +- .../log-ingestor/tests/test_ingestion_job.rs | 26 ++--- 18 files changed, 181 insertions(+), 84 deletions(-) create mode 100644 components/clp-rust-utils/src/types.rs diff --git a/Cargo.lock b/Cargo.lock index 82650ece36..5da6411e9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -150,7 +150,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -161,7 +161,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -874,7 +874,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -892,6 +892,7 @@ dependencies = [ "aws-sdk-sqs", "brotli", "hex", + "non-empty-string", "num_enum", "rmp-serde", "secrecy", @@ -1151,7 +1152,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.111", ] [[package]] @@ -1162,7 +1163,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1171,6 +1172,17 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" +[[package]] +name = "delegate" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "082a24a9967533dc5d743c602157637116fc1b52806d694a5a45e6f32567fcdd" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "der" version = "0.6.1" @@ -1209,7 +1221,7 @@ checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1220,7 +1232,7 @@ checksum = "ef941ded77d15ca19b40374869ac6000af1c9f2a4c0f3d4c70926287e6364a8f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1242,7 +1254,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.111", "unicode-xid", ] @@ -1266,7 +1278,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1331,7 +1343,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1495,7 +1507,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -2141,6 +2153,8 @@ dependencies = [ "clap", "clp-rust-utils", "const_format", + "log", + "non-empty-string", "secrecy", "serde", "serde_json", @@ -2173,7 +2187,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -2187,7 +2201,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -2198,7 +2212,7 @@ checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -2209,7 +2223,7 @@ checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" dependencies = [ "macro_magic_core", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -2351,7 +2365,17 @@ dependencies = [ "macro_magic", "proc-macro2", "quote", - "syn", + "syn 2.0.111", +] + +[[package]] +name = "non-empty-string" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "260010741d90def3ca7c4b5ec0bbe28c26d0c775d3c7d291b38642514136677a" +dependencies = [ + "delegate", + "serde", ] [[package]] @@ -2434,7 +2458,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -3095,7 +3119,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -3154,7 +3178,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -3192,7 +3216,7 @@ checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -3382,7 +3406,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn", + "syn 2.0.111", ] [[package]] @@ -3405,7 +3429,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn", + "syn 2.0.111", "tokio", "url", ] @@ -3542,6 +3566,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.111" @@ -3567,7 +3602,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -3605,7 +3640,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -3707,7 +3742,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -3859,7 +3894,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -3937,7 +3972,7 @@ checksum = "0e48cea23f68d1f78eb7bc092881b6bb88d3d6b5b7e6234f6f9c911da1ffb221" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -4061,7 +4096,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn", + "syn 2.0.111", ] [[package]] @@ -4162,7 +4197,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.111", "wasm-bindgen-shared", ] @@ -4493,7 +4528,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", "synstructure", ] @@ -4514,7 +4549,7 @@ checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -4534,7 +4569,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", "synstructure", ] @@ -4574,5 +4609,5 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] diff --git a/components/clp-rust-utils/Cargo.toml b/components/clp-rust-utils/Cargo.toml index b8a0a3cad1..76ccddcbd7 100644 --- a/components/clp-rust-utils/Cargo.toml +++ b/components/clp-rust-utils/Cargo.toml @@ -9,6 +9,7 @@ aws-sdk-s3 = "1.110.0" aws-sdk-sqs = "1.88.0" brotli = "8.0.2" num_enum = "0.7.5" +non-empty-string = { version = "0.2.6", features = ["serde"] } rmp-serde = "1.3.0" secrecy = { version = "0.10.3", features = ["serde"] } serde = { version = "1.0.228", features = ["derive"] } diff --git a/components/clp-rust-utils/src/clp_config/package.rs b/components/clp-rust-utils/src/clp_config/package.rs index b879fc8db7..76bb12f15a 100644 --- a/components/clp-rust-utils/src/clp_config/package.rs +++ b/components/clp-rust-utils/src/clp_config/package.rs @@ -1,3 +1,9 @@ +use std::sync::LazyLock; + +use non_empty_string::NonEmptyString; + +use crate::types::non_empty_string::ExpectedNonEmpty; + pub mod config; pub mod credentials; @@ -5,5 +11,5 @@ pub mod credentials; pub const DEFAULT_CONFIG_FILE_PATH: &str = "etc/clp-config.yaml"; pub const DEFAULT_CREDENTIALS_FILE_PATH: &str = "etc/credentials.yaml"; -/// Mirror of constants in `clp_py_utils.clp_config`. -pub const DEFAULT_DATASET_NAME: &str = "default"; +pub static DEFAULT_DATASET_NAME: LazyLock = + LazyLock::new(|| NonEmptyString::from_static_str("default")); diff --git a/components/clp-rust-utils/src/clp_config/s3_config.rs b/components/clp-rust-utils/src/clp_config/s3_config.rs index 15e70cdae2..af1c251f7a 100644 --- a/components/clp-rust-utils/src/clp_config/s3_config.rs +++ b/components/clp-rust-utils/src/clp_config/s3_config.rs @@ -1,11 +1,12 @@ +use non_empty_string::NonEmptyString; use serde::{Deserialize, Serialize}; /// Represents the configuration for connecting to an S3 bucket. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct S3Config { - pub bucket: String, + pub bucket: NonEmptyString, pub region_code: String, - pub key_prefix: String, + pub key_prefix: NonEmptyString, pub aws_authentication: AwsAuthentication, } diff --git a/components/clp-rust-utils/src/job_config/clp_io_config.rs b/components/clp-rust-utils/src/job_config/clp_io_config.rs index 2ca7d2cc7c..10b9e1f331 100644 --- a/components/clp-rust-utils/src/job_config/clp_io_config.rs +++ b/components/clp-rust-utils/src/job_config/clp_io_config.rs @@ -1,3 +1,4 @@ +use non_empty_string::NonEmptyString; use serde::Serialize; use crate::clp_config::S3Config; @@ -26,16 +27,16 @@ pub struct S3InputConfig { #[serde(flatten)] pub s3_config: S3Config, - pub keys: Option>, - pub dataset: Option, - pub timestamp_key: Option, + pub keys: Option>, + pub dataset: Option, + pub timestamp_key: Option, pub unstructured: bool, } /// Represents CLP output config. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct OutputConfig { - pub tags: Option>, + pub tags: Option>, pub target_archive_size: u64, pub target_dictionaries_size: u64, pub target_encoded_file_size: u64, diff --git a/components/clp-rust-utils/src/job_config/ingestion.rs b/components/clp-rust-utils/src/job_config/ingestion.rs index 19f0034ab2..d31cb1f870 100644 --- a/components/clp-rust-utils/src/job_config/ingestion.rs +++ b/components/clp-rust-utils/src/job_config/ingestion.rs @@ -1,4 +1,5 @@ pub mod s3 { + use non_empty_string::NonEmptyString; use serde::{Deserialize, Serialize}; /// Base configuration for ingesting logs from S3. @@ -8,18 +9,18 @@ pub mod s3 { pub region: String, /// The S3 bucket to ingest from. - pub bucket_name: String, + pub bucket_name: NonEmptyString, /// The S3 key prefix to ingest from. - pub key_prefix: String, + pub key_prefix: NonEmptyString, /// The dataset to ingest into. Defaults to `None` (which uses the default dataset). #[serde(default)] - pub dataset: Option, + pub dataset: Option, /// The optional key for extracting timestamps from object metadata. Defaults to `None`. #[serde(default)] - pub timestamp_key: Option, + pub timestamp_key: Option, /// Whether to treat the ingested objects as unstructured logs. Defaults to `false`. #[serde(default = "default_unstructured")] @@ -27,7 +28,7 @@ pub mod s3 { /// Tags to apply on the compressed archives. Defaults to `None`. #[serde(default)] - pub tags: Option>, + pub tags: Option>, } /// Configuration for a SQS listener job. @@ -38,7 +39,7 @@ pub mod s3 { /// The SQS queue URL to poll for S3 event notifications. The given queue must be dedicated /// to this ingestion job. - pub queue_url: String, + pub queue_url: NonEmptyString, } /// Configuration for a S3 scanner job. @@ -54,7 +55,7 @@ pub mod s3 { /// The key to ingest after. If specified, only objects with keys lexicographically greater /// than this value will be ingested. Defaults to `None`. #[serde(default)] - pub start_after: Option, + pub start_after: Option, } const fn default_unstructured() -> bool { diff --git a/components/clp-rust-utils/src/lib.rs b/components/clp-rust-utils/src/lib.rs index 95660acc08..723dc44a4a 100644 --- a/components/clp-rust-utils/src/lib.rs +++ b/components/clp-rust-utils/src/lib.rs @@ -5,5 +5,6 @@ pub mod job_config; pub mod s3; pub mod serde; pub mod sqs; +pub mod types; pub use error::Error; diff --git a/components/clp-rust-utils/src/s3.rs b/components/clp-rust-utils/src/s3.rs index 7d70d5e5f8..78f321f327 100644 --- a/components/clp-rust-utils/src/s3.rs +++ b/components/clp-rust-utils/src/s3.rs @@ -1,10 +1,11 @@ mod client; pub use client::create_new_client; +use non_empty_string::NonEmptyString; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct ObjectMetadata { - pub bucket: String, - pub key: String, + pub bucket: NonEmptyString, + pub key: NonEmptyString, pub size: u64, } diff --git a/components/clp-rust-utils/src/sqs/event/s3.rs b/components/clp-rust-utils/src/sqs/event/s3.rs index 959e260212..70049c2c9e 100644 --- a/components/clp-rust-utils/src/sqs/event/s3.rs +++ b/components/clp-rust-utils/src/sqs/event/s3.rs @@ -1,3 +1,4 @@ +use non_empty_string::NonEmptyString; use serde::Deserialize; /// Represents an Amazon S3 event notification message. @@ -28,11 +29,11 @@ pub struct Entity { #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] pub struct Bucket { - pub name: String, + pub name: NonEmptyString, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] pub struct Object { - pub key: String, + pub key: NonEmptyString, pub size: u64, } diff --git a/components/clp-rust-utils/src/types.rs b/components/clp-rust-utils/src/types.rs new file mode 100644 index 0000000000..8e31d20a49 --- /dev/null +++ b/components/clp-rust-utils/src/types.rs @@ -0,0 +1,32 @@ +pub mod non_empty_string { + use non_empty_string::NonEmptyString; + + /// Trait for creating [`NonEmptyString`] instances from an expected non-empty string. + pub trait ExpectedNonEmpty { + /// # Returns + /// + /// A [`NonEmptyString`] of the provided static string slice. + /// + /// # Panics + /// + /// Panics if the provided static string slice is empty. + #[must_use] + fn from_static_str(str: &'static str) -> NonEmptyString { + NonEmptyString::new(str.to_owned()).expect("static &str shouldn't be empty") + } + + /// # Returns + /// + /// A [`NonEmptyString`] of the provided string. + /// + /// # Panics + /// + /// Panics if the provided string is empty. + #[must_use] + fn from_string(str: String) -> NonEmptyString { + NonEmptyString::new(str).expect("string shouldn't be empty") + } + } + + impl ExpectedNonEmpty for NonEmptyString {} +} diff --git a/components/clp-rust-utils/tests/clp_config_test.rs b/components/clp-rust-utils/tests/clp_config_test.rs index 06cf1915d9..f58384ea16 100644 --- a/components/clp-rust-utils/tests/clp_config_test.rs +++ b/components/clp-rust-utils/tests/clp_config_test.rs @@ -2,15 +2,17 @@ use clp_rust_utils::{ clp_config::{AwsAuthentication, AwsCredentials, S3Config}, job_config::{ClpIoConfig, InputConfig, OutputConfig, S3InputConfig}, serde::BrotliMsgpack, + types::non_empty_string::ExpectedNonEmpty, }; +use non_empty_string::NonEmptyString; use serde_json::Value; #[test] fn test_clp_io_config_serialization() { let s3_config = S3Config { - bucket: "yscope".into(), + bucket: NonEmptyString::from_static_str("yscope"), region_code: "us-east-2".into(), - key_prefix: "sample-logs/cockroachdb.clp.zst".into(), + key_prefix: NonEmptyString::from_static_str("sample-logs/cockroachdb.clp.zst"), aws_authentication: AwsAuthentication::Credentials { credentials: AwsCredentials { access_key_id: "ACCESS_KEY_ID".into(), @@ -23,8 +25,8 @@ fn test_clp_io_config_serialization() { config: S3InputConfig { s3_config, keys: None, - dataset: Some("test-dataset".into()), - timestamp_key: Some("timestamp".into()), + dataset: Some(NonEmptyString::from_static_str("test-dataset")), + timestamp_key: Some(NonEmptyString::from_static_str("timestamp")), unstructured: false, }, }, diff --git a/components/log-ingestor/Cargo.toml b/components/log-ingestor/Cargo.toml index bb8e55e059..d95c8b09fd 100644 --- a/components/log-ingestor/Cargo.toml +++ b/components/log-ingestor/Cargo.toml @@ -12,6 +12,7 @@ aws-sdk-sqs = "1.89.0" clap = { version = "4.5.51", features = ["derive"] } clp-rust-utils = { path = "../clp-rust-utils" } const_format = "0.2.35" +non-empty-string = { version = "0.2.6", features = ["serde"] } secrecy = "0.10.3" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.145" @@ -23,6 +24,7 @@ tracing-appender = "0.2.2" tracing-subscriber = { version = "0.3.20", features = ["json", "env-filter", "fmt", "std"] } uuid = { version = "1.18.1", features = ["v4"] } thiserror = "2.0.17" +log = "0.4.29" [dev-dependencies] serial_test = "3.2.0" diff --git a/components/log-ingestor/src/compression/compression_job_submitter.rs b/components/log-ingestor/src/compression/compression_job_submitter.rs index dafadc3edd..5ed432921e 100644 --- a/components/log-ingestor/src/compression/compression_job_submitter.rs +++ b/components/log-ingestor/src/compression/compression_job_submitter.rs @@ -76,7 +76,7 @@ impl CompressionJobSubmitter { ingestion_job_config .dataset .clone() - .unwrap_or_else(|| DEFAULT_DATASET_NAME.to_string()), + .unwrap_or_else(|| DEFAULT_DATASET_NAME.clone()), ), timestamp_key: ingestion_job_config.timestamp_key.clone(), unstructured: ingestion_job_config.unstructured, diff --git a/components/log-ingestor/src/ingestion_job/s3_scanner.rs b/components/log-ingestor/src/ingestion_job/s3_scanner.rs index b345c6188e..2416c041c0 100644 --- a/components/log-ingestor/src/ingestion_job/s3_scanner.rs +++ b/components/log-ingestor/src/ingestion_job/s3_scanner.rs @@ -3,6 +3,7 @@ use std::time::Duration; use anyhow::Result; use aws_sdk_s3::Client; use clp_rust_utils::{job_config::ingestion::s3::S3ScannerConfig, s3::ObjectMetadata}; +use non_empty_string::NonEmptyString; use tokio::{select, sync::mpsc}; use tokio_util::sync::CancellationToken; use uuid::Uuid; @@ -20,6 +21,7 @@ struct Task> { scanning_interval: Duration, config: S3ScannerConfig, sender: mpsc::Sender, + start_after: Option, } impl> Task { @@ -89,13 +91,14 @@ impl> Task { /// * Forwards /// [`aws_sdk_s3::operation::list_objects_v2::builders::ListObjectsV2FluentBuilder::send`]'s /// return values on failure. + /// * Forwards [`NonEmptyString::new`]'s return values on failure. pub async fn scan_once(&mut self) -> Result { let client = self.s3_client_manager.get().await?; let response = client .list_objects_v2() .bucket(self.config.base.bucket_name.as_str()) .prefix(self.config.base.key_prefix.as_str()) - .set_start_after(self.config.start_after.clone()) + .set_start_after(self.start_after.clone()) .send() .await?; let Some(contents) = response.contents else { @@ -111,12 +114,13 @@ impl> Task { } let object_metadata = ObjectMetadata { bucket: self.config.base.bucket_name.clone(), - key: key.clone(), + key: NonEmptyString::new(key.clone()) + .map_err(|_| anyhow::anyhow!("An empty key is received."))?, size: size.try_into()?, }; tracing::info!(object = ? object_metadata, "Scanned new object metadata on S3."); self.sender.send(object_metadata).await?; - self.config.start_after = Some(key); + self.start_after = Some(key); } Ok(response.is_truncated.unwrap_or(false)) @@ -156,11 +160,13 @@ impl S3Scanner { sender: mpsc::Sender, ) -> Self { let scanning_interval = Duration::from_secs(u64::from(config.scanning_interval_sec)); + let start_after = config.start_after.clone().map(Into::into); let task = Task { s3_client_manager, scanning_interval, config, sender, + start_after, }; let cancel_token = CancellationToken::new(); let child_cancel_token = cancel_token.clone(); diff --git a/components/log-ingestor/src/ingestion_job_manager.rs b/components/log-ingestor/src/ingestion_job_manager.rs index 9ba586811a..9ecc0bc813 100644 --- a/components/log-ingestor/src/ingestion_job_manager.rs +++ b/components/log-ingestor/src/ingestion_job_manager.rs @@ -12,6 +12,7 @@ use clp_rust_utils::{ job_config::ingestion::s3::{BaseConfig, S3ScannerConfig, SqsListenerConfig}, s3::ObjectMetadata, }; +use non_empty_string::NonEmptyString; use tokio::sync::{Mutex, mpsc}; use uuid::Uuid; @@ -293,9 +294,9 @@ struct IngestionJobTableEntry { ingestion_job: IngestionJob, listener: Listener, region: String, - bucket_name: String, - key_prefix: String, - dataset: Option, + bucket_name: NonEmptyString, + key_prefix: NonEmptyString, + dataset: Option, } /// # Returns: diff --git a/components/log-ingestor/tests/aws_config.rs b/components/log-ingestor/tests/aws_config.rs index e939346475..6d69be00e0 100644 --- a/components/log-ingestor/tests/aws_config.rs +++ b/components/log-ingestor/tests/aws_config.rs @@ -1,4 +1,5 @@ use anyhow::{Result, anyhow}; +use non_empty_string::NonEmptyString; /// Default AWS configuration for local testing with `LocalStack`. const DEFAULT_AWS_ENDPOINT_URL: &str = "http://127.0.0.1:4566"; @@ -14,8 +15,8 @@ pub struct AwsConfig { pub secret_access_key: String, pub region: String, pub account_id: String, - pub bucket_name: String, - pub queue_name: String, + pub bucket_name: NonEmptyString, + pub queue_name: NonEmptyString, } impl AwsConfig { @@ -42,8 +43,8 @@ impl AwsConfig { /// /// Returns an error if: /// - /// * `CLP_LOG_INGESTOR_S3_BUCKET` environment variable is not set. - /// * `CLP_LOG_INGESTOR_SQS_QUEUE` environment variable is not set. + /// * `CLP_LOG_INGESTOR_S3_BUCKET` environment variable is not set, or set to an empty string. + /// * `CLP_LOG_INGESTOR_SQS_QUEUE` environment variable is not set, or set to an empty string. pub fn from_env() -> Result { let endpoint = std::env::var("AWS_ENDPOINT_URL") .unwrap_or_else(|_| DEFAULT_AWS_ENDPOINT_URL.to_string()); @@ -68,8 +69,10 @@ impl AwsConfig { secret_access_key, region, account_id, - bucket_name, - queue_name, + bucket_name: NonEmptyString::new(bucket_name) + .map_err(|_| anyhow!("bucket name must not be empty"))?, + queue_name: NonEmptyString::new(queue_name) + .map_err(|_| anyhow!("queue name must not be empty"))?, }) } } diff --git a/components/log-ingestor/tests/test_compression_listener.rs b/components/log-ingestor/tests/test_compression_listener.rs index 6337e66ba2..e87d3eaed3 100644 --- a/components/log-ingestor/tests/test_compression_listener.rs +++ b/components/log-ingestor/tests/test_compression_listener.rs @@ -2,8 +2,9 @@ use std::{sync::Arc, time::Duration}; use anyhow::Result; use async_trait::async_trait; -use clp_rust_utils::s3::ObjectMetadata; +use clp_rust_utils::{s3::ObjectMetadata, types::non_empty_string::ExpectedNonEmpty}; use log_ingestor::compression::{Buffer, BufferSubmitter, DEFAULT_LISTENER_CAPACITY, Listener}; +use non_empty_string::NonEmptyString; use tokio::sync::{Mutex, mpsc}; const TEST_OBJECT_SIZE: u64 = 1024; @@ -50,8 +51,8 @@ async fn send_to_listener(objects: Vec, sender: mpsc::Sender Vec { (0..count) .map(|i| ObjectMetadata { - bucket: bucket_name.to_string(), - key: format!("object-{i}"), + bucket: NonEmptyString::from_string(bucket_name.to_string()), + key: NonEmptyString::from_string(format!("object-{i}")), size: TEST_OBJECT_SIZE, }) .collect() diff --git a/components/log-ingestor/tests/test_ingestion_job.rs b/components/log-ingestor/tests/test_ingestion_job.rs index 6568ae0dfa..506e69c425 100644 --- a/components/log-ingestor/tests/test_ingestion_job.rs +++ b/components/log-ingestor/tests/test_ingestion_job.rs @@ -7,11 +7,13 @@ use aws_config::AwsConfig; use clp_rust_utils::{ job_config::ingestion::s3::{BaseConfig, S3ScannerConfig, SqsListenerConfig}, s3::ObjectMetadata, + types::non_empty_string::ExpectedNonEmpty, }; use log_ingestor::{ aws_client_manager::{S3ClientWrapper, SqsClientWrapper}, ingestion_job::SqsListener, }; +use non_empty_string::NonEmptyString; use tokio::sync::mpsc; use uuid::Uuid; @@ -87,15 +89,15 @@ async fn receive_object_metadata( /// * A vector of received S3 object metadata. async fn upload_and_receive( s3_client: aws_sdk_s3::Client, - bucket: String, - prefix: String, + bucket: NonEmptyString, + prefix: NonEmptyString, num_objects_to_create: usize, receiver: mpsc::Receiver, ) -> (Vec, Vec) { let objects_to_create: Vec<_> = (0..num_objects_to_create) .map(|idx| ObjectMetadata { bucket: bucket.clone(), - key: format!("{prefix}/{idx:05}.log"), + key: NonEmptyString::from_string(format!("{prefix}/{idx:05}.log")), size: 16, }) .collect(); @@ -124,13 +126,13 @@ async fn upload_and_receive( /// The keys are formatted as `{uuid}.log`, where `uuid` is a randomly generated v4 UUID. async fn upload_noise_objects( s3_client: aws_sdk_s3::Client, - bucket: String, + bucket: NonEmptyString, num_objects_to_create: usize, ) { let objects_to_create: Vec<_> = (0..num_objects_to_create) .map(|_| ObjectMetadata { bucket: bucket.clone(), - key: format!("{}.log", Uuid::new_v4()), + key: NonEmptyString::from_string(format!("{}.log", Uuid::new_v4())), size: 16, }) .collect(); @@ -142,9 +144,9 @@ async fn upload_noise_objects( /// # Returns /// -/// A unique testing prefix for S3 object keys. The prefix is formatted as `test-{job_id}/`. -fn get_testing_prefix(job_id: &Uuid) -> String { - format!("test-{job_id}") +/// A unique testing prefix for S3 object keys. The prefix is formatted as `test-{job_id}`. +fn get_testing_prefix_as_non_empty_string(job_id: &Uuid) -> NonEmptyString { + NonEmptyString::from_string(format!("test-{job_id}")) } #[tokio::test] @@ -152,7 +154,7 @@ fn get_testing_prefix(job_id: &Uuid) -> String { #[ignore = "Requires LocalStack or AWS environment"] async fn test_sqs_listener() -> Result<()> { let job_id = Uuid::new_v4(); - let prefix = get_testing_prefix(&job_id); + let prefix = get_testing_prefix_as_non_empty_string(&job_id); let aws_config = AwsConfig::from_env()?; @@ -165,12 +167,12 @@ async fn test_sqs_listener() -> Result<()> { .await; let sqs_listener_config = SqsListenerConfig { - queue_url: format!( + queue_url: NonEmptyString::from_string(format!( "{}/{}/{}", aws_config.endpoint.as_str(), aws_config.account_id.as_str(), aws_config.queue_name.as_str() - ), + )), base: BaseConfig { region: aws_config.region.clone(), bucket_name: aws_config.bucket_name.clone(), @@ -233,7 +235,7 @@ async fn test_sqs_listener() -> Result<()> { #[ignore = "Requires LocalStack or AWS environment"] async fn test_s3_scanner() -> Result<()> { let job_id = Uuid::new_v4(); - let prefix = get_testing_prefix(&job_id); + let prefix = get_testing_prefix_as_non_empty_string(&job_id); let aws_config = AwsConfig::from_env()?; From 96ef44c065f931cf55d37c51e2aa97d7e6da5437 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 15 Dec 2025 03:47:58 +0800 Subject: [PATCH 33/91] feat(core): Add runtime CA bundle path detection for `CurlDownloadHandler`; Resolve clang-tidy violations in `clp/CurlDownloadHandler.{cpp,hpp}`. (#1751) Co-authored-by: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> --- .../core/src/clp/CurlDownloadHandler.cpp | 39 ++++++++++++++++++- .../core/src/clp/CurlDownloadHandler.hpp | 22 +++++++++++ taskfiles/lint.yaml | 2 - 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/components/core/src/clp/CurlDownloadHandler.cpp b/components/core/src/clp/CurlDownloadHandler.cpp index e6910f8e83..f580051aa7 100644 --- a/components/core/src/clp/CurlDownloadHandler.cpp +++ b/components/core/src/clp/CurlDownloadHandler.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include #include @@ -17,6 +19,7 @@ #include #include "ErrorCode.hpp" +#include "Platform.hpp" namespace clp { CurlDownloadHandler::CurlDownloadHandler( @@ -42,7 +45,14 @@ CurlDownloadHandler::CurlDownloadHandler( } // Set up src url - m_easy_handle.set_option(CURLOPT_URL, src_url.data()); + m_easy_handle.set_option(CURLOPT_URL, std::string{src_url}.c_str()); + + // Set up CA bundle path + if (auto const optional_ca_bundle_path{get_host_ca_bundle_path()}; + optional_ca_bundle_path.has_value()) + { + m_easy_handle.set_option(CURLOPT_CAINFO, optional_ca_bundle_path->c_str()); + } // Set up progress callback m_easy_handle.set_option(CURLOPT_XFERINFOFUNCTION, progress_callback); @@ -115,4 +125,31 @@ CurlDownloadHandler::CurlDownloadHandler( // Set up failure on HTTP error reponse m_easy_handle.set_option(CURLOPT_FAILONERROR, static_cast(true)); } + +auto CurlDownloadHandler::get_host_ca_bundle_path() -> std::optional { + if constexpr (Platform::MacOs == cCurrentPlatform) { + return std::nullopt; + } + + static constexpr std::string_view cDebianCaBundlePath{"/etc/ssl/certs/ca-certificates.crt"}; + static constexpr std::string_view cCentOsCaBundlePath{"/etc/pki/tls/certs/ca-bundle.crt"}; + + // Read-only operation. No multithreaded context. + // NOLINTNEXTLINE(concurrency-mt-unsafe) + if (auto const* path{std::getenv("CURL_CA_BUNDLE")}; nullptr != path) { + return path; + } + // Read-only operation. No multithreaded context. + // NOLINTNEXTLINE(concurrency-mt-unsafe) + if (auto const* path{std::getenv("SSL_CERT_FILE")}; nullptr != path) { + return path; + } + if (std::filesystem::exists(cDebianCaBundlePath)) { + return std::string{cDebianCaBundlePath}; + } + if (std::filesystem::exists(cCentOsCaBundlePath)) { + return std::string{cCentOsCaBundlePath}; + } + return std::nullopt; +} } // namespace clp diff --git a/components/core/src/clp/CurlDownloadHandler.hpp b/components/core/src/clp/CurlDownloadHandler.hpp index e7c4b73a88..3b9ef1ed31 100644 --- a/components/core/src/clp/CurlDownloadHandler.hpp +++ b/components/core/src/clp/CurlDownloadHandler.hpp @@ -11,6 +11,7 @@ #include #include +#include #include "CurlEasyHandle.hpp" #include "CurlStringList.hpp" @@ -91,6 +92,27 @@ class CurlDownloadHandler { [[nodiscard]] auto perform() -> CURLcode { return m_easy_handle.perform(); } private: + /** + * Locates the certificate authority (CA) bundle file available on the current host. + * + * This method performs a runtime lookup to provide a predictable CA bundle path for libcurl on + * Linux-based systems. It is intended to handle cases where the runtime environment differs + * from the build environment. macOS typically use their native system certificate stores and + * are not searched by this function. + * + * Resolution order (first match wins): + * 1. Environment variables, checked in order: `CURL_CA_BUNDLE`, then `SSL_CERT_FILE`. If the + * variable is set and points to an existing, readable file, that path is returned. + * 2. Known distribution-specific default bundle locations (checked in order): + * - Debian / Ubuntu: `/etc/ssl/certs/ca-certificates.crt` + * - CentOS / RHEL / Fedora: `/etc/pki/tls/certs/ca-bundle.crt` + * + * @return Absolute path to the discovered CA bundle file. + * @return std::nullopt if no CA bundle file could be located. The caller should handle this + * case by relying on libcurl's built-in defaults. + */ + [[nodiscard]] static auto get_host_ca_bundle_path() -> std::optional; + CurlEasyHandle m_easy_handle; CurlStringList m_http_headers; std::shared_ptr m_error_msg_buf; diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 32d13ef618..23103e8175 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -245,8 +245,6 @@ tasks: - "clp/clp/utils.cpp" - "clp/clp/utils.hpp" - "clp/CommandLineArgumentsBase.hpp" - - "clp/CurlDownloadHandler.cpp" - - "clp/CurlDownloadHandler.hpp" - "clp/CurlEasyHandle.hpp" - "clp/CurlStringList.hpp" - "clp/database_utils.cpp" From 633d213d20a6ec1dd536c052d88147b1667d61e7 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Mon, 15 Dec 2025 14:57:13 -0500 Subject: [PATCH 34/91] feat(webui): Move timestamp presets into AntD date picker. (#1734) --- .../TimeRangeInput/TimeDateInput.tsx | 30 +++-- .../TimeRangePanel/index.module.css | 7 ++ .../TimeRangeInput/TimeRangePanel/index.tsx | 74 ++++++++++++ .../TimeRangeInput/index.module.css | 13 +-- .../SearchControls/TimeRangeInput/index.tsx | 110 +++++++++--------- 5 files changed, 162 insertions(+), 72 deletions(-) create mode 100644 components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/TimeRangePanel/index.module.css create mode 100644 components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/TimeRangePanel/index.tsx diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/TimeDateInput.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/TimeDateInput.tsx index f90f1dd5e1..edb856d4d6 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/TimeDateInput.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/TimeDateInput.tsx @@ -7,9 +7,10 @@ import { } from "./utils"; -interface TimeDateInputProps { +interface TimeDateInputProps extends React.InputHTMLAttributes { value: string; [DATE_RANGE_PROP_KEY]: DATE_RANGE_POSITION; + isPickerOpen?: boolean; } /** @@ -19,20 +20,35 @@ interface TimeDateInputProps { * @return */ const TimeDateInput = (props: TimeDateInputProps) => { - const {value, [DATE_RANGE_PROP_KEY]: dateRange} = props; - const timeRangeOption = useSearchStore((state) => state.timeRangeOption); + const { + value, + [DATE_RANGE_PROP_KEY]: dateRange, + onChange, + isPickerOpen = false, + ...restProps + } = props; - const displayText = timeRangeOption === TIME_RANGE_OPTION.CUSTOM ? + const timeRangeOption = useSearchStore((state) => state.timeRangeOption); + const updateTimeRangeOption = useSearchStore((state) => state.updateTimeRangeOption); + const displayText = (timeRangeOption === TIME_RANGE_OPTION.CUSTOM || isPickerOpen) ? value : TIME_RANGE_DISPLAY_TEXT_MAP[timeRangeOption][dateRange]; + const handleChange = (e: React.ChangeEvent) => { + if (timeRangeOption !== TIME_RANGE_OPTION.CUSTOM) { + updateTimeRangeOption(TIME_RANGE_OPTION.CUSTOM); + } + onChange?.(e); + }; + return ( + {...restProps} + value={displayText} + onChange={handleChange}/> ); }; +export type {TimeDateInputProps}; export default TimeDateInput; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/TimeRangePanel/index.module.css b/components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/TimeRangePanel/index.module.css new file mode 100644 index 0000000000..1556137a01 --- /dev/null +++ b/components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/TimeRangePanel/index.module.css @@ -0,0 +1,7 @@ +.panelContainer { + display: flex; +} + +.presetList { + min-width: 150px; +} diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/TimeRangePanel/index.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/TimeRangePanel/index.tsx new file mode 100644 index 0000000000..e7df2d58cc --- /dev/null +++ b/components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/TimeRangePanel/index.tsx @@ -0,0 +1,74 @@ +import React from "react"; + +import { + ConfigProvider, + Menu, +} from "antd"; + +import useSearchStore from "../../../SearchState/index"; +import { + TIME_RANGE_OPTION, + TIME_RANGE_OPTION_DAYJS_MAP, + TIME_RANGE_OPTION_NAMES, +} from "../utils"; +import styles from "./index.module.css"; + + +interface TimeRangePanelProps { + panelNode: React.ReactNode; + onClose: () => void; +} + +/** + * Renders a time range selection panel with preset options and a custom date picker. + * + * @param props + * @param props.panelNode The date picker panel node + * @param props.onClose Callback to close the panel + * @return + */ +const TimeRangePanel = ({panelNode, onClose}: TimeRangePanelProps) => { + const updateTimeRange = useSearchStore((state) => state.updateTimeRange); + const updateTimeRangeOption = useSearchStore((state) => state.updateTimeRangeOption); + const timeRangeOption = useSearchStore((state) => state.timeRangeOption); + + const handlePresetClick = (option: TIME_RANGE_OPTION) => { + updateTimeRangeOption(option); + TIME_RANGE_OPTION_DAYJS_MAP[option]() + .then((dates) => { + updateTimeRange([dates[0].utc(true), + dates[1].utc(true)]); + onClose(); + }) + .catch((e: unknown) => { + console.error("Failed to set time range:", e); + }); + }; + + return ( +
+ + { + handlePresetClick(key as TIME_RANGE_OPTION); + }} + > + {TIME_RANGE_OPTION_NAMES + .filter((option) => option !== TIME_RANGE_OPTION.CUSTOM) + .map((option) => ( + + {option} + + ))} + + + {panelNode} +
+ ); +}; + +export default TimeRangePanel; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/index.module.css b/components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/index.module.css index 27116fdc7d..ef97f5c683 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/index.module.css +++ b/components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/index.module.css @@ -2,18 +2,7 @@ display: inline-flex; } -.timeRangeInputContainer :global(.ant-select) { - min-width: 150px; -} - -/* Makes border flush with range picker */ -.customSelected :global(.ant-select-selector) { - border-top-right-radius: 0 !important; - border-bottom-right-radius: 0 !important; -} - .rangePicker { - border-top-left-radius: 0; - border-bottom-left-radius: 0; min-width: 350px; } + diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/index.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/index.tsx index d66bbebd52..08b126ba9d 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/TimeRangeInput/index.tsx @@ -1,13 +1,14 @@ -import {useCallback} from "react"; +import React, { + type ComponentProps, + useCallback, + useState, +} from "react"; import {CLP_QUERY_ENGINES} from "@webui/common/config"; import { - ConfigProvider, DatePicker, - Select, - theme, + GetProp, } from "antd"; -import dayjs from "dayjs"; import {SETTINGS_QUERY_ENGINE} from "../../../../config"; import useSearchStore from "../../SearchState/index"; @@ -16,14 +17,20 @@ import {PRESTO_SQL_INTERFACE} from "../../SearchState/Presto/typings"; import {SEARCH_UI_STATE} from "../../SearchState/typings"; import styles from "./index.module.css"; import TimeRangeFooter from "./Presto/TimeRangeFooter"; -import TimeDateInput from "./TimeDateInput"; +import TimeDateInput, {type TimeDateInputProps} from "./TimeDateInput"; +import TimeRangePanel from "./TimeRangePanel/index"; import { isValidDateRange, TIME_RANGE_OPTION, - TIME_RANGE_OPTION_NAMES, } from "./utils"; +type RangePickerDates = Parameters< + NonNullable< + GetProp, "onCalendarChange"> + >>[0]; + + /** * Renders controls for selecting a time range for queries. By default, the component is * a select dropdown with a list of preset time ranges. If the user selects "Custom", @@ -35,35 +42,55 @@ const TimeRangeInput = () => { const { timeRange, updateTimeRange, - timeRangeOption, updateTimeRangeOption, searchUiState, } = useSearchStore(); - const {token} = theme.useToken(); + const [isOpen, setIsOpen] = useState(false); const sqlInterface = usePrestoSearchState((state) => state.sqlInterface); - const isPrestoGuided = SETTINGS_QUERY_ENGINE === CLP_QUERY_ENGINES.PRESTO && - sqlInterface === PRESTO_SQL_INTERFACE.GUIDED; - - const handleSelectChange = (newTimeRangeOption: TIME_RANGE_OPTION) => { - updateTimeRangeOption(newTimeRangeOption); - }; + const isPrestoGuided = + SETTINGS_QUERY_ENGINE === CLP_QUERY_ENGINES.PRESTO && + sqlInterface === PRESTO_SQL_INTERFACE.GUIDED; - const handleRangePickerChange = ( - dates: [dayjs.Dayjs | null, dayjs.Dayjs | null] | null + const handleRangePickerChange = useCallback(( + dates: RangePickerDates ) => { if (!isValidDateRange(dates)) { return; } + // User manually changed the dates, switch to CUSTOM + updateTimeRangeOption(TIME_RANGE_OPTION.CUSTOM); + // Treat range picker selection as UTC by dropping any timezone offset supplied by antd. updateTimeRange([ dates[0].utc(true), dates[1].utc(true), ]); + }, [updateTimeRange, + updateTimeRangeOption]); + + const handleOpenChange = (open: boolean) => { + setIsOpen(open); }; + const handleClose = useCallback(() => { + setIsOpen(false); + }, []); + + const panelRender = useCallback((panelNode: React.ReactNode) => ( + + ), [handleClose]); + + const inputComponent = useCallback((props: TimeDateInputProps) => ( + + ), [isOpen]); + const renderFooter = useCallback(() => { if (false === isPrestoGuided) { return null; @@ -76,45 +103,22 @@ const TimeRangeInput = () => {
-