From 88ccce7d7ad82fc1cdd5458fe9f3914bfd0db2d7 Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Mon, 17 Mar 2025 09:58:45 -0400 Subject: [PATCH 01/49] Updated log-surgeon to use simulation branch. --- components/core/submodules/log-surgeon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/submodules/log-surgeon b/components/core/submodules/log-surgeon index 895f46489b..b42fae9a70 160000 --- a/components/core/submodules/log-surgeon +++ b/components/core/submodules/log-surgeon @@ -1 +1 @@ -Subproject commit 895f46489b1911ab3b3aac3202afd56c96e8cd98 +Subproject commit b42fae9a70972c1119190b27334e6d81e03ead54 From 8aa0350d83faae176e893a36d4f9e7a14d7b14d9 Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Mon, 17 Mar 2025 10:19:20 -0400 Subject: [PATCH 02/49] Update schema to have all timestamps and capture. --- components/core/config/schemas.txt | 59 ++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index e0b777859d..315196709e 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -1,19 +1,62 @@ -// Delimiters -delimiters: \t\r\n!"#$%&'\(\)\*,:;<>?@\[\]\^_`\{\|\}~ - // Timestamps (using the `timestamp` keyword) +// E.g. 2015-01-31T15:50:45.392 +timestamp:\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}.\d{3} +// E.g. 2015-01-31T15:50:45,392 +timestamp:\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2},\d{3} +// E.g. [2015-01-31T15:50:45 +timestamp:\[\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2} +// E.g. [20170106-16:56:41] +timestamp:\[\d{4}\d{2}\d{2}\-\d{2}:\d{2}:\d{2}\] // E.g. 2015-01-31 15:50:45,392 +// E.g. INFO [main] 2015-01-31 15:50:45,085 +timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2},\d{3} // E.g. 2015-01-31 15:50:45.392 +timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}.\d{3} +// E.g. [2015-01-31 15:50:45,085] +timestamp:\[\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2},\d{3}\] // E.g. 2015-01-31 15:50:45 -timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3}){0,1} -// E.g. [20150131-15:50:45] -timestamp:\[\d{8}\-\d{2}:\d{2}:\d{2}\] +// E.g. Started POST /api/v3/internal/allowed for 127.0.0.1 at 2017-06-18 00:20:44 +// E.g. update-alternatives 2015-01-31 15:50:45 +timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2} +// E.g. Start-Date: 2015-01-31 15:50:45 +timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2} +// E.g. 2015/01/31 15:50:45 +timestamp:\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} +// E.g. 15/01/31 15:50:45 +timestamp:\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} +// E.g. 150131 9:50:45 +timestamp:\d{2}\d{2}\d{2} [ 0-9]{2}:\d{2}:\d{2} +// E.g. 01 Jan 2016 15:50:17,085 +timestamp:\d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2},\d{3} +// E.g. Jan 01, 2016 3:50:17 PM +timestamp:[A-Z][a-z]{2} \d{2}, \d{4} [ 0-9]{2}:\d{2}:\d{2} [AP]M +// E.g. January 31, 2015 15:50 +timestamp:[A-Z][a-z]+ \d{2}, \d{4} \d{2}:\d{2} +// E.g. E [31/Jan/2015:15:50:45 +// E.g. localhost - - [01/Jan/2016:15:50:17 +// E.g. 192.168.4.5 - - [01/Jan/2016:15:50:17 +timestamp:\[\d{2}/[A-Z][a-z]{2}/\d{4}:\d{2}:\d{2}:\d{2} +// E.g. 192.168.4.5 - - [01/01/2016:15:50:17 +timestamp:\[\d{2}/\d{2}/\d{4}:\d{2}:\d{2}:\d{2} +// E.g. ERROR: apport (pid 4557) Sun Jan 1 15:50:45 2015 +timestamp:[A-Z][a-z]{2} [A-Z][a-z]{2} [ 0-9]{2} \d{2}:\d{2}:\d{2} \d{4} +// E.g. <<<2016-11-10 03:02:29:936 +timestamp:\<\<\<\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}:\d{3} +// E.g. Jan 21 11:56:42 +timestamp:[A-Z][a-z]{2} \d{2} \d{2}:\d{2}:\d{2} +// E.g. 01-21 11:56:42.392 +timestamp:\d{2}\-\d{2} \d{2}:\d{2}:\d{2}.\d{3} +// E.g. 2016-05-08 11:34:04.083464 +timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}.\d{6} + +// Delimiters +delimiters: \t\r\n!"#$%&'\(\)\*,:;<>?@\[\]\^_`\{\|\}~= // Specially-encoded variables (using the `int` and `float` keywords) int:\-{0,1}[0-9]+ float:\-{0,1}[0-9]+\.[0-9]+ -// Dictionary variables +// Dictionary variables (the way `equals` and `hasNumber` are written only work if `=` is a delim) hex:[a-fA-F]+ +equals:.*=(?.*[a-zA-Z0-9].*) hasNumber:.*\d.* -equals:.*=.*[a-zA-Z0-9].* From b3f217b62cedda1fdef57fad7a1c280b30597992 Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Mon, 24 Mar 2025 09:53:24 -0400 Subject: [PATCH 03/49] Testing new log surgeon. --- components/core/config/schemas.txt | 2 +- components/core/src/clp/Grep.cpp | 8 ++-- components/core/src/clp/Utils.cpp | 38 +++++++++---------- .../clp/streaming_archive/writer/Archive.cpp | 12 +++--- .../core/tests/test-ParserWithUserSchema.cpp | 4 +- 5 files changed, 32 insertions(+), 32 deletions(-) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index 315196709e..1dd3811db6 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -50,7 +50,7 @@ timestamp:\d{2}\-\d{2} \d{2}:\d{2}:\d{2}.\d{3} timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}.\d{6} // Delimiters -delimiters: \t\r\n!"#$%&'\(\)\*,:;<>?@\[\]\^_`\{\|\}~= +delimiters: \t\r\n!"#$%&'\(\)\*,:;\<\>\?@\[\]\^_`\{\|\}~= // Specially-encoded variables (using the `int` and `float` keywords) int:\-{0,1}[0-9]+ diff --git a/components/core/src/clp/Grep.cpp b/components/core/src/clp/Grep.cpp index a53679edf5..d4e2059e93 100644 --- a/components/core/src/clp/Grep.cpp +++ b/components/core/src/clp/Grep.cpp @@ -871,16 +871,16 @@ bool Grep::get_bounds_of_next_potential_var( } // TODO: use a set so its faster // auto const& set = search_token.m_type_ids_set; - // if (set.find(static_cast(log_surgeon::SymbolID::TokenUncaughtStringID)) + // if (set.find(static_cast(log_surgeon::SymbolId::TokenUncaughtStringID)) // == set.end() - // && set.find(static_cast(log_surgeon::SymbolID::TokenEndID)) + // && set.find(static_cast(log_surgeon::SymbolId::TokenEndID)) // == set.end()) // { // is_var = true; // } auto const& type = search_token.m_type_ids_ptr->at(0); - if (type != static_cast(log_surgeon::SymbolID::TokenUncaughtStringID) - && type != static_cast(log_surgeon::SymbolID::TokenEndID)) + if (type != static_cast(log_surgeon::SymbolId::TokenUncaughtStringID) + && type != static_cast(log_surgeon::SymbolId::TokenEndID)) { is_var = true; } diff --git a/components/core/src/clp/Utils.cpp b/components/core/src/clp/Utils.cpp index 1c0fc05cab..81e751e2d9 100644 --- a/components/core/src/clp/Utils.cpp +++ b/components/core/src/clp/Utils.cpp @@ -125,7 +125,6 @@ void load_lexer_from_file( bool reverse, log_surgeon::lexers::ByteLexer& lexer ) { - log_surgeon::SchemaParser sp; std::unique_ptr schema_ast = log_surgeon::SchemaParser::try_schema_file(schema_file_path); if (!lexer.m_symbol_id.empty()) { @@ -134,43 +133,43 @@ void load_lexer_from_file( // cTokenEnd and cTokenUncaughtString never need to be added as a rule to the lexer as they are // not parsed - lexer.m_symbol_id[log_surgeon::cTokenEnd] = static_cast(log_surgeon::SymbolID::TokenEndID); + lexer.m_symbol_id[log_surgeon::cTokenEnd] = static_cast(log_surgeon::SymbolId::TokenEnd); lexer.m_symbol_id[log_surgeon::cTokenUncaughtString] - = static_cast(log_surgeon::SymbolID::TokenUncaughtStringID); + = static_cast(log_surgeon::SymbolId::TokenUncaughtString); // cTokenInt, cTokenFloat, cTokenFirstTimestamp, and cTokenNewlineTimestamp each have unknown // rule(s) until specified by the user so can't be explicitly added and are done by looping over // schema_vars (user schema) - lexer.m_symbol_id[log_surgeon::cTokenInt] = static_cast(log_surgeon::SymbolID::TokenIntId); + lexer.m_symbol_id[log_surgeon::cTokenInt] = static_cast(log_surgeon::SymbolId::TokenInt); lexer.m_symbol_id[log_surgeon::cTokenFloat] - = static_cast(log_surgeon::SymbolID::TokenFloatId); + = static_cast(log_surgeon::SymbolId::TokenFloat); lexer.m_symbol_id[log_surgeon::cTokenFirstTimestamp] - = static_cast(log_surgeon::SymbolID::TokenFirstTimestampId); + = static_cast(log_surgeon::SymbolId::TokenFirstTimestamp); lexer.m_symbol_id[log_surgeon::cTokenNewlineTimestamp] - = static_cast(log_surgeon::SymbolID::TokenNewlineTimestampId); + = static_cast(log_surgeon::SymbolId::TokenNewlineTimestamp); // cTokenNewline is not added in schema_vars and can be explicitly added as '\n' to catch the // end of non-timestamped log messages lexer.m_symbol_id[log_surgeon::cTokenNewline] - = static_cast(log_surgeon::SymbolID::TokenNewlineId); + = static_cast(log_surgeon::SymbolId::TokenNewline); - lexer.m_id_symbol[static_cast(log_surgeon::SymbolID::TokenEndID)] = log_surgeon::cTokenEnd; - lexer.m_id_symbol[static_cast(log_surgeon::SymbolID::TokenUncaughtStringID)] + lexer.m_id_symbol[static_cast(log_surgeon::SymbolId::TokenEnd)] = log_surgeon::cTokenEnd; + lexer.m_id_symbol[static_cast(log_surgeon::SymbolId::TokenUncaughtString)] = log_surgeon::cTokenUncaughtString; - lexer.m_id_symbol[static_cast(log_surgeon::SymbolID::TokenIntId)] = log_surgeon::cTokenInt; - lexer.m_id_symbol[static_cast(log_surgeon::SymbolID::TokenFloatId)] + lexer.m_id_symbol[static_cast(log_surgeon::SymbolId::TokenInt)] = log_surgeon::cTokenInt; + lexer.m_id_symbol[static_cast(log_surgeon::SymbolId::TokenFloat)] = log_surgeon::cTokenFloat; - lexer.m_id_symbol[static_cast(log_surgeon::SymbolID::TokenFirstTimestampId)] + lexer.m_id_symbol[static_cast(log_surgeon::SymbolId::TokenFirstTimestamp)] = log_surgeon::cTokenFirstTimestamp; - lexer.m_id_symbol[static_cast(log_surgeon::SymbolID::TokenNewlineTimestampId)] + lexer.m_id_symbol[static_cast(log_surgeon::SymbolId::TokenNewlineTimestamp)] = log_surgeon::cTokenNewlineTimestamp; - lexer.m_id_symbol[static_cast(log_surgeon::SymbolID::TokenNewlineId)] + lexer.m_id_symbol[static_cast(log_surgeon::SymbolId::TokenNewline)] = log_surgeon::cTokenNewline; lexer.add_rule( lexer.m_symbol_id["newLine"], std::move(std::make_unique>( + log_surgeon::finite_automata::ByteNfaState>>( log_surgeon::finite_automata::RegexASTLiteral< - log_surgeon::finite_automata::RegexNFAByteState>('\n') + log_surgeon::finite_automata::ByteNfaState>('\n') )) ); @@ -201,7 +200,7 @@ void load_lexer_from_file( // transform '.' from any-character into any non-delimiter character rule->m_regex_ptr->remove_delimiters_from_wildcard(delimiters); - bool is_possible_input[log_surgeon::cUnicodeMax] = {false}; + std::array is_possible_input{}; rule->m_regex_ptr->set_possible_inputs_to_true(is_possible_input); bool contains_delimiter = false; uint32_t delimiter_name; @@ -241,7 +240,8 @@ void load_lexer_from_file( lexer.add_rule(lexer.m_symbol_id[rule->m_name], std::move(rule->m_regex_ptr)); } if (reverse) { - lexer.generate_reverse(); + // TODO: This isn't used anymore for the new search, supporting it here is a waste of time + //lexer.generate_reverse(); } else { lexer.generate(); } diff --git a/components/core/src/clp/streaming_archive/writer/Archive.cpp b/components/core/src/clp/streaming_archive/writer/Archive.cpp index 688cf3c23e..8a13882060 100644 --- a/components/core/src/clp/streaming_archive/writer/Archive.cpp +++ b/components/core/src/clp/streaming_archive/writer/Archive.cpp @@ -364,8 +364,8 @@ void Archive::write_msg_using_schema(LogEventView const& log_view) { log_surgeon::Token& token = log_output_buffer->get_mutable_token(i); int token_type = token.m_type_ids_ptr->at(0); if (log_output_buffer->has_delimiters() && (timestamp_pattern != nullptr || i > 1) - && token_type != static_cast(log_surgeon::SymbolID::TokenUncaughtStringID) - && token_type != static_cast(log_surgeon::SymbolID::TokenNewlineId)) + && token_type != static_cast(log_surgeon::SymbolId::TokenUncaughtString) + && token_type != static_cast(log_surgeon::SymbolId::TokenNewline)) { m_logtype_dict_entry.add_constant(token.get_delimiter(), 0, 1); if (token.m_start_pos == token.m_buffer_size - 1) { @@ -375,12 +375,12 @@ void Archive::write_msg_using_schema(LogEventView const& log_view) { } } switch (token_type) { - case static_cast(log_surgeon::SymbolID::TokenNewlineId): - case static_cast(log_surgeon::SymbolID::TokenUncaughtStringID): { + case static_cast(log_surgeon::SymbolId::TokenNewline): + case static_cast(log_surgeon::SymbolId::TokenUncaughtString): { m_logtype_dict_entry.add_constant(token.to_string(), 0, token.get_length()); break; } - case static_cast(log_surgeon::SymbolID::TokenIntId): { + case static_cast(log_surgeon::SymbolId::TokenInt): { encoded_variable_t encoded_var; if (!EncodedVariableInterpreter::convert_string_to_representable_integer_var( token.to_string(), @@ -397,7 +397,7 @@ void Archive::write_msg_using_schema(LogEventView const& log_view) { m_encoded_vars.push_back(encoded_var); break; } - case static_cast(log_surgeon::SymbolID::TokenFloatId): { + case static_cast(log_surgeon::SymbolId::TokenFloat): { encoded_variable_t encoded_var; if (!EncodedVariableInterpreter::convert_string_to_representable_float_var( token.to_string(), diff --git a/components/core/tests/test-ParserWithUserSchema.cpp b/components/core/tests/test-ParserWithUserSchema.cpp index 3689c69e85..2c3420f966 100644 --- a/components/core/tests/test-ParserWithUserSchema.cpp +++ b/components/core/tests/test-ParserWithUserSchema.cpp @@ -170,7 +170,7 @@ TEST_CASE("Test forward lexer", "[Search]") { Token token; auto error_code = forward_lexer.scan(parser_input_buffer, token); REQUIRE(error_code == log_surgeon::ErrorCode::Success); - while (token.m_type_ids_ptr->at(0) != static_cast(log_surgeon::SymbolID::TokenEndID)) { + while (token.m_type_ids_ptr->at(0) != static_cast(log_surgeon::SymbolId::TokenEndID)) { SPDLOG_INFO("token:" + token.to_string() + "\n"); SPDLOG_INFO( "token.m_type_ids->back():" @@ -194,7 +194,7 @@ TEST_CASE("Test reverse lexer", "[Search]") { Token token; auto error_code = reverse_lexer.scan(parser_input_buffer, token); REQUIRE(error_code == log_surgeon::ErrorCode::Success); - while (token.m_type_ids_ptr->at(0) != static_cast(log_surgeon::SymbolID::TokenEndID)) { + while (token.m_type_ids_ptr->at(0) != static_cast(log_surgeon::SymbolId::TokenEndID)) { SPDLOG_INFO("token:" + token.to_string() + "\n"); SPDLOG_INFO( "token.m_type_ids->back():" From ea09df0e0a3dc8f1a7b15477971225ad32280fdb Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Mon, 24 Mar 2025 09:56:14 -0400 Subject: [PATCH 04/49] Testing log surgeon x2. --- components/core/submodules/log-surgeon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/submodules/log-surgeon b/components/core/submodules/log-surgeon index b42fae9a70..1d7cec56f4 160000 --- a/components/core/submodules/log-surgeon +++ b/components/core/submodules/log-surgeon @@ -1 +1 @@ -Subproject commit b42fae9a70972c1119190b27334e6d81e03ead54 +Subproject commit 1d7cec56f4550102a837857096f68ccacad7ab06 From 551447fa6475c93f70b979b816e92d46614c87c8 Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Mon, 31 Mar 2025 09:36:48 -0400 Subject: [PATCH 05/49] Add timers. --- components/core/CMakeLists.txt | 2 +- components/core/src/clp/Profiler.hpp | 20 +++++++++++++++++-- .../core/src/clp/clp/FileCompressor.cpp | 5 ++--- components/core/src/clp/clp/run.cpp | 9 +++++++++ components/core/submodules/log-surgeon | 2 +- 5 files changed, 31 insertions(+), 7 deletions(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index d8b0352a45..95bbdf9597 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -34,7 +34,7 @@ string(LENGTH "${CMAKE_SOURCE_DIR}/" SOURCE_PATH_SIZE) add_definitions("-DSOURCE_PATH_SIZE=${SOURCE_PATH_SIZE}") # Profiling options -add_definitions(-DPROF_ENABLED=0) +add_definitions(-DPROF_ENABLED=1) # Compile-in debug logging statements #add_definitions(-DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_DEBUG) diff --git a/components/core/src/clp/Profiler.hpp b/components/core/src/clp/Profiler.hpp index f93dec0708..cc5a7f8e0f 100644 --- a/components/core/src/clp/Profiler.hpp +++ b/components/core/src/clp/Profiler.hpp @@ -41,11 +41,19 @@ class Profiler { // Types enum class ContinuousMeasurementIndex : size_t { Compression = 0, - ParseLogFile, Search, Length }; enum class FragmentedMeasurementIndex : size_t { + Watch, + ParseLogFile, + Parse, + GetNext, + ProcessState, + ScanWhileLoop, + TokenCreation, + ProcessCharWatch, + ProcessChar, Length }; @@ -54,12 +62,20 @@ class Profiler { static constexpr auto cContinuousMeasurementEnabled = []() { std::array enabled{}; enabled[enum_to_underlying_type(ContinuousMeasurementIndex::Compression)] = true; - enabled[enum_to_underlying_type(ContinuousMeasurementIndex::ParseLogFile)] = true; enabled[enum_to_underlying_type(ContinuousMeasurementIndex::Search)] = true; return enabled; }(); static constexpr auto cFragmentedMeasurementEnabled = []() { std::array enabled{}; + enabled[enum_to_underlying_type(FragmentedMeasurementIndex::Watch)] = false; + enabled[enum_to_underlying_type(FragmentedMeasurementIndex::ParseLogFile)] = false; + enabled[enum_to_underlying_type(FragmentedMeasurementIndex::Parse)] = false; + enabled[enum_to_underlying_type(FragmentedMeasurementIndex::GetNext)] = false; + enabled[enum_to_underlying_type(FragmentedMeasurementIndex::ProcessState)] = false; + enabled[enum_to_underlying_type(FragmentedMeasurementIndex::ScanWhileLoop)] = false; + enabled[enum_to_underlying_type(FragmentedMeasurementIndex::TokenCreation)] = false; + enabled[enum_to_underlying_type(FragmentedMeasurementIndex::ProcessCharWatch)] = true; + enabled[enum_to_underlying_type(FragmentedMeasurementIndex::ProcessChar)] = true; return enabled; }(); diff --git a/components/core/src/clp/clp/FileCompressor.cpp b/components/core/src/clp/clp/FileCompressor.cpp index 9898602cc8..1c5336d7f0 100644 --- a/components/core/src/clp/clp/FileCompressor.cpp +++ b/components/core/src/clp/clp/FileCompressor.cpp @@ -119,7 +119,7 @@ bool FileCompressor::compress_file( std::string file_name = std::filesystem::canonical(file_to_compress.get_path()).string(); PROFILER_SPDLOG_INFO("Start parsing {}", file_name) - Profiler::start_continuous_measurement(); + Profiler::start_fragmented_measurement(); m_file_reader.open(file_to_compress.get_path()); @@ -186,8 +186,7 @@ bool FileCompressor::compress_file( m_file_reader.close(); - Profiler::stop_continuous_measurement(); - LOG_CONTINUOUS_MEASUREMENT(Profiler::ContinuousMeasurementIndex::ParseLogFile) + Profiler::stop_fragmented_measurement(); PROFILER_SPDLOG_INFO("Done parsing {}", file_name) return succeeded; diff --git a/components/core/src/clp/clp/run.cpp b/components/core/src/clp/clp/run.cpp index 5a3b0eb270..d7cb7ee631 100644 --- a/components/core/src/clp/clp/run.cpp +++ b/components/core/src/clp/clp/run.cpp @@ -151,6 +151,15 @@ int run(int argc, char const* argv[]) { Profiler::stop_continuous_measurement(); LOG_CONTINUOUS_MEASUREMENT(Profiler::ContinuousMeasurementIndex::Compression) + LOG_FRAGMENTED_MEASUREMENT(Profiler::FragmentedMeasurementIndex::Watch) + LOG_FRAGMENTED_MEASUREMENT(Profiler::FragmentedMeasurementIndex::Parse) + LOG_FRAGMENTED_MEASUREMENT(Profiler::FragmentedMeasurementIndex::GetNext) + LOG_FRAGMENTED_MEASUREMENT(Profiler::FragmentedMeasurementIndex::ProcessState) + LOG_FRAGMENTED_MEASUREMENT(Profiler::FragmentedMeasurementIndex::ScanWhileLoop) + LOG_FRAGMENTED_MEASUREMENT(Profiler::FragmentedMeasurementIndex::TokenCreation) + LOG_FRAGMENTED_MEASUREMENT(Profiler::FragmentedMeasurementIndex::ParseLogFile) + LOG_FRAGMENTED_MEASUREMENT(Profiler::FragmentedMeasurementIndex::ProcessCharWatch) + LOG_FRAGMENTED_MEASUREMENT(Profiler::FragmentedMeasurementIndex::ProcessChar) return 0; } diff --git a/components/core/submodules/log-surgeon b/components/core/submodules/log-surgeon index 1d7cec56f4..df0bb632a6 160000 --- a/components/core/submodules/log-surgeon +++ b/components/core/submodules/log-surgeon @@ -1 +1 @@ -Subproject commit 1d7cec56f4550102a837857096f68ccacad7ab06 +Subproject commit df0bb632a6b3e6a038f33bc21e4f6c1a277e78dd From ee3f68241643eba204220c743de26d54103815c7 Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Sun, 6 Apr 2025 20:07:41 -0400 Subject: [PATCH 06/49] Switch timers. --- components/core/src/clp/Profiler.hpp | 12 ++++++------ components/core/submodules/log-surgeon | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/core/src/clp/Profiler.hpp b/components/core/src/clp/Profiler.hpp index cc5a7f8e0f..a355df17a4 100644 --- a/components/core/src/clp/Profiler.hpp +++ b/components/core/src/clp/Profiler.hpp @@ -61,21 +61,21 @@ class Profiler { // NOTE: We use lambdas so that we can programmatically initialize the constexpr array static constexpr auto cContinuousMeasurementEnabled = []() { std::array enabled{}; - enabled[enum_to_underlying_type(ContinuousMeasurementIndex::Compression)] = true; - enabled[enum_to_underlying_type(ContinuousMeasurementIndex::Search)] = true; + enabled[enum_to_underlying_type(ContinuousMeasurementIndex::Compression)] = false; + enabled[enum_to_underlying_type(ContinuousMeasurementIndex::Search)] = false; return enabled; }(); static constexpr auto cFragmentedMeasurementEnabled = []() { std::array enabled{}; - enabled[enum_to_underlying_type(FragmentedMeasurementIndex::Watch)] = false; enabled[enum_to_underlying_type(FragmentedMeasurementIndex::ParseLogFile)] = false; enabled[enum_to_underlying_type(FragmentedMeasurementIndex::Parse)] = false; enabled[enum_to_underlying_type(FragmentedMeasurementIndex::GetNext)] = false; enabled[enum_to_underlying_type(FragmentedMeasurementIndex::ProcessState)] = false; enabled[enum_to_underlying_type(FragmentedMeasurementIndex::ScanWhileLoop)] = false; - enabled[enum_to_underlying_type(FragmentedMeasurementIndex::TokenCreation)] = false; - enabled[enum_to_underlying_type(FragmentedMeasurementIndex::ProcessCharWatch)] = true; - enabled[enum_to_underlying_type(FragmentedMeasurementIndex::ProcessChar)] = true; + enabled[enum_to_underlying_type(FragmentedMeasurementIndex::Watch)] = true; + enabled[enum_to_underlying_type(FragmentedMeasurementIndex::TokenCreation)] = true; + enabled[enum_to_underlying_type(FragmentedMeasurementIndex::ProcessCharWatch)] = false; + enabled[enum_to_underlying_type(FragmentedMeasurementIndex::ProcessChar)] = false; return enabled; }(); diff --git a/components/core/submodules/log-surgeon b/components/core/submodules/log-surgeon index df0bb632a6..5de8b6f8d7 160000 --- a/components/core/submodules/log-surgeon +++ b/components/core/submodules/log-surgeon @@ -1 +1 @@ -Subproject commit df0bb632a6b3e6a038f33bc21e4f6c1a277e78dd +Subproject commit 5de8b6f8d77d69b81ae9e4c12eec2f2abb3b9c28 From 042a4f6d1e5865bc52cf1f4315c3e979559948aa Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Wed, 23 Apr 2025 08:44:54 -0400 Subject: [PATCH 07/49] Make make dictionaries compilable. --- .../core/src/clp/make_dictionaries_readable/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/core/src/clp/make_dictionaries_readable/CMakeLists.txt b/components/core/src/clp/make_dictionaries_readable/CMakeLists.txt index 9779d137f4..f3db1a51b3 100644 --- a/components/core/src/clp/make_dictionaries_readable/CMakeLists.txt +++ b/components/core/src/clp/make_dictionaries_readable/CMakeLists.txt @@ -17,11 +17,15 @@ set( ../LogTypeDictionaryReader.hpp ../ParsedMessage.cpp ../ParsedMessage.hpp + ../Profiler.hpp + ../Profiler.cpp ../ReaderInterface.cpp ../ReaderInterface.hpp ../ReadOnlyMemoryMappedFile.cpp ../ReadOnlyMemoryMappedFile.hpp ../spdlog_with_specializations.hpp + ../Stopwatch.cpp + ../Stopwatch.hpp ../streaming_compression/Decompressor.hpp ../streaming_compression/passthrough/Decompressor.cpp ../streaming_compression/passthrough/Decompressor.hpp From 4b534272da1d769c167eeb1c3676b7e42766b3c0 Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Wed, 25 Jun 2025 02:18:22 -0400 Subject: [PATCH 08/49] Lint and try to reduce the number of jobs during deps:core to avoid memory crash. --- components/core/src/clp/Utils.cpp | 14 ++++++++------ taskfiles/deps/main.yaml | 1 + 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/components/core/src/clp/Utils.cpp b/components/core/src/clp/Utils.cpp index 81e751e2d9..2e4207074d 100644 --- a/components/core/src/clp/Utils.cpp +++ b/components/core/src/clp/Utils.cpp @@ -166,11 +166,13 @@ void load_lexer_from_file( lexer.add_rule( lexer.m_symbol_id["newLine"], - std::move(std::make_unique>( - log_surgeon::finite_automata::RegexASTLiteral< - log_surgeon::finite_automata::ByteNfaState>('\n') - )) + std::move( + std::make_unique>( + log_surgeon::finite_automata::RegexASTLiteral< + log_surgeon::finite_automata::ByteNfaState>('\n') + ) + ) ); for (auto const& delimiters_ast : schema_ast->m_delimiters) { @@ -241,7 +243,7 @@ void load_lexer_from_file( } if (reverse) { // TODO: This isn't used anymore for the new search, supporting it here is a waste of time - //lexer.generate_reverse(); + // lexer.generate_reverse(); } else { lexer.generate(); } diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 16f999df94..bed28b028e 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -39,6 +39,7 @@ tasks: - "mkdir -p '{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}'" - task: "yscope-dev-utils:cmake:install-deps-and-generate-settings" vars: + JOBS: 1 CMAKE_SETTINGS_DIR: "{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}" DEP_TASK: "core-all-parallel" - task: "utils:combine-core-checksum-files" From 3d27fdfaeb17942b2817c4e0202dd0227c489ada Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Wed, 25 Jun 2025 04:01:12 -0400 Subject: [PATCH 09/49] Remove JOBS from task, the correct way is to set the parallel tasks variable. --- taskfiles/deps/main.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index bed28b028e..16f999df94 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -39,7 +39,6 @@ tasks: - "mkdir -p '{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}'" - task: "yscope-dev-utils:cmake:install-deps-and-generate-settings" vars: - JOBS: 1 CMAKE_SETTINGS_DIR: "{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}" DEP_TASK: "core-all-parallel" - task: "utils:combine-core-checksum-files" From 018eccbe3b40a3145e6ed262f26d487afc6bb1b1 Mon Sep 17 00:00:00 2001 From: davidlion Date: Wed, 9 Jul 2025 21:57:55 -0400 Subject: [PATCH 10/49] Remove components/core/submodules/log-surgeon. --- components/core/submodules/log-surgeon | 1 - 1 file changed, 1 deletion(-) delete mode 160000 components/core/submodules/log-surgeon diff --git a/components/core/submodules/log-surgeon b/components/core/submodules/log-surgeon deleted file mode 160000 index b42fae9a70..0000000000 --- a/components/core/submodules/log-surgeon +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b42fae9a70972c1119190b27334e6d81e03ead54 From 5971025db3a7ef3ab11f1a53e0bc3e5a4a84a0f7 Mon Sep 17 00:00:00 2001 From: davidlion Date: Wed, 9 Jul 2025 21:59:17 -0400 Subject: [PATCH 11/49] Unset PROF_ENABLED. --- components/core/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 3e4c612b07..a0889845b2 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -48,7 +48,7 @@ string(LENGTH "${CMAKE_SOURCE_DIR}/" SOURCE_PATH_SIZE) add_definitions("-DSOURCE_PATH_SIZE=${SOURCE_PATH_SIZE}") # Profiling options -add_definitions(-DPROF_ENABLED=1) +add_definitions(-DPROF_ENABLED=0) # Compile-in debug logging statements #add_definitions(-DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_DEBUG) From b932b6a9571344990b20d74819813069e6bff6d4 Mon Sep 17 00:00:00 2001 From: davidlion Date: Wed, 9 Jul 2025 23:10:07 -0400 Subject: [PATCH 12/49] Get clp building with log surgeon locally. --- components/core/src/clp/Grep.cpp | 10 +++++++--- components/core/src/clp/Utils.cpp | 3 ++- taskfiles/deps/main.yaml | 26 ++++++++++++++++++++++++-- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/components/core/src/clp/Grep.cpp b/components/core/src/clp/Grep.cpp index d968a933a0..f34aecc0d6 100644 --- a/components/core/src/clp/Grep.cpp +++ b/components/core/src/clp/Grep.cpp @@ -866,7 +866,11 @@ bool Grep::get_bounds_of_next_potential_var( string_reader.open(value.substr(begin_pos, end_pos - begin_pos)); parser_input_buffer.read_if_safe(reader_wrapper); forward_lexer.reset(); - forward_lexer.scan(parser_input_buffer, search_token); + auto [err, token] = forward_lexer.scan(parser_input_buffer); + if (log_surgeon::ErrorCode::Success != err) { + return false; + } + search_token = SearchToken{token.value()}; search_token.m_type_ids_set.insert(search_token.m_type_ids_ptr->at(0)); } // TODO: use a set so its faster @@ -879,8 +883,8 @@ bool Grep::get_bounds_of_next_potential_var( // is_var = true; // } auto const& type = search_token.m_type_ids_ptr->at(0); - if (type != static_cast(log_surgeon::SymbolId::TokenUncaughtStringID) - && type != static_cast(log_surgeon::SymbolId::TokenEndID)) + if (type != static_cast(log_surgeon::SymbolId::TokenUncaughtString) + && type != static_cast(log_surgeon::SymbolId::TokenEnd)) { is_var = true; } diff --git a/components/core/src/clp/Utils.cpp b/components/core/src/clp/Utils.cpp index 2e4207074d..8e65d6c486 100644 --- a/components/core/src/clp/Utils.cpp +++ b/components/core/src/clp/Utils.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -178,7 +179,7 @@ void load_lexer_from_file( for (auto const& delimiters_ast : schema_ast->m_delimiters) { auto* delimiters_ptr = dynamic_cast(delimiters_ast.get()); if (delimiters_ptr != nullptr) { - lexer.add_delimiters(delimiters_ptr->m_delimiters); + lexer.set_delimiters(delimiters_ptr->m_delimiters); } } vector delimiters; diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index bed28b028e..dcaf276555 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -19,6 +19,7 @@ vars: # Library names G_FMT_LIB_NAME: "fmt" + G_GSL_LIB_NAME: "Microsoft.GSL" # Antlr G_ANTLR_VERSION: "4.13.2" @@ -67,6 +68,7 @@ tasks: - task: "date" - task: "fmt" - task: "log-surgeon" + - task: "microsoft.gsl" - task: "mongocxx" - task: "nlohmann_json" - task: "simdjson" @@ -221,15 +223,35 @@ tasks: log-surgeon: internal: true run: "once" + deps: + - "fmt" + - "microsoft.gsl" cmds: - task: "utils:install-remote-cmake-lib" vars: CMAKE_GEN_ARGS: + - "-C {{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}/{{.G_FMT_LIB_NAME}}.cmake" + - "-C {{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}/{{.G_GSL_LIB_NAME}}.cmake" - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-Dlog_surgeon_BUILD_TESTING=OFF" LIB_NAME: "log_surgeon" - TARBALL_SHA256: "169b56714b620c3f8f07bf7975aba8573b9bd25e423e5c4f0cc9b52450a2403d" - TARBALL_URL: "https://github.com/y-scope/log-surgeon/archive/f801a3f.tar.gz" + TARBALL_SHA256: "9b723033ed4d6ea1e6831014d1d0921982399cecd076c9418b4dd780e72743ad" + TARBALL_URL: "https://github.com/y-scope/log-surgeon/archive/c954252.tar.gz" + + microsoft.gsl: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DGSL_TEST=OFF" + LIB_NAME: "{{.G_GSL_LIB_NAME}}" + TARBALL_SHA256: "f0e32cb10654fea91ad56bde89170d78cfbf4363ee0b01d8f097de2ba49f6ce9" + TARBALL_URL: "https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.tar.gz" mongocxx: internal: true From 50cfd39f61bdf680cf3055c786ce0879d3872ad1 Mon Sep 17 00:00:00 2001 From: davidlion Date: Wed, 9 Jul 2025 23:56:49 -0400 Subject: [PATCH 13/49] Unit tests build, but fail with possible logical errors. --- .../core/tests/test-ParserWithUserSchema.cpp | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/components/core/tests/test-ParserWithUserSchema.cpp b/components/core/tests/test-ParserWithUserSchema.cpp index 50fa1a7ca7..bb6251b5ac 100644 --- a/components/core/tests/test-ParserWithUserSchema.cpp +++ b/components/core/tests/test-ParserWithUserSchema.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include "../src/clp/clp/run.hpp" #include "../src/clp/GlobalMySQLMetadataDB.hpp" @@ -19,18 +20,15 @@ using clp::FileReader; using clp::load_lexer_from_file; using clp::LogSurgeonReader; using log_surgeon::DelimiterStringAST; -using log_surgeon::LALR1Parser; using log_surgeon::lexers::ByteLexer; using log_surgeon::LogParser; using log_surgeon::ParserAST; using log_surgeon::SchemaAST; -using log_surgeon::SchemaParser; using log_surgeon::SchemaVarAST; using log_surgeon::Token; std::unique_ptr generate_schema_ast(std::string const& schema_file) { - SchemaParser schema_parser; - std::unique_ptr schema_ast = SchemaParser::try_schema_file(schema_file); + std::unique_ptr schema_ast = log_surgeon::SchemaParser::try_schema_file(schema_file); REQUIRE(schema_ast.get() != nullptr); return schema_ast; } @@ -167,17 +165,19 @@ TEST_CASE("Test forward lexer", "[Search]") { log_surgeon::ParserInputBuffer parser_input_buffer; parser_input_buffer.read_if_safe(reader_wrapper); forward_lexer.reset(); - Token token; - auto error_code = forward_lexer.scan(parser_input_buffer, token); + auto [error_code, opt_token] = forward_lexer.scan(parser_input_buffer); REQUIRE(error_code == log_surgeon::ErrorCode::Success); - while (token.m_type_ids_ptr->at(0) != static_cast(log_surgeon::SymbolId::TokenEndID)) { + Token token{opt_token.value()}; + REQUIRE(error_code == log_surgeon::ErrorCode::Success); + while (token.m_type_ids_ptr->at(0) != static_cast(log_surgeon::SymbolId::TokenEnd)) { SPDLOG_INFO("token:" + token.to_string() + "\n"); SPDLOG_INFO( "token.m_type_ids->back():" + forward_lexer.m_id_symbol[token.m_type_ids_ptr->back()] + "\n" ); - error_code = forward_lexer.scan(parser_input_buffer, token); + auto [error_code, opt_token] = forward_lexer.scan(parser_input_buffer); REQUIRE(error_code == log_surgeon::ErrorCode::Success); + token = opt_token.value(); } } @@ -191,16 +191,17 @@ TEST_CASE("Test reverse lexer", "[Search]") { log_surgeon::ParserInputBuffer parser_input_buffer; parser_input_buffer.read_if_safe(reader_wrapper); reverse_lexer.reset(); - Token token; - auto error_code = reverse_lexer.scan(parser_input_buffer, token); + auto [error_code, opt_token] = reverse_lexer.scan(parser_input_buffer); REQUIRE(error_code == log_surgeon::ErrorCode::Success); - while (token.m_type_ids_ptr->at(0) != static_cast(log_surgeon::SymbolId::TokenEndID)) { + Token token{opt_token.value()}; + while (token.m_type_ids_ptr->at(0) != static_cast(log_surgeon::SymbolId::TokenEnd)) { SPDLOG_INFO("token:" + token.to_string() + "\n"); SPDLOG_INFO( "token.m_type_ids->back():" + reverse_lexer.m_id_symbol[token.m_type_ids_ptr->back()] + "\n" ); - error_code = reverse_lexer.scan(parser_input_buffer, token); + auto [error_code, opt_token] = reverse_lexer.scan(parser_input_buffer); REQUIRE(error_code == log_surgeon::ErrorCode::Success); + token = opt_token.value(); } } From 2083aa4394eb9f2620289b8b79d8dd40d55a7ab6 Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Mon, 14 Jul 2025 12:38:42 -0400 Subject: [PATCH 14/49] Remove duplicate REQUIRE check. --- components/core/tests/test-ParserWithUserSchema.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/core/tests/test-ParserWithUserSchema.cpp b/components/core/tests/test-ParserWithUserSchema.cpp index bb6251b5ac..830badea3a 100644 --- a/components/core/tests/test-ParserWithUserSchema.cpp +++ b/components/core/tests/test-ParserWithUserSchema.cpp @@ -168,7 +168,6 @@ TEST_CASE("Test forward lexer", "[Search]") { auto [error_code, opt_token] = forward_lexer.scan(parser_input_buffer); REQUIRE(error_code == log_surgeon::ErrorCode::Success); Token token{opt_token.value()}; - REQUIRE(error_code == log_surgeon::ErrorCode::Success); while (token.m_type_ids_ptr->at(0) != static_cast(log_surgeon::SymbolId::TokenEnd)) { SPDLOG_INFO("token:" + token.to_string() + "\n"); SPDLOG_INFO( From e778979f156abd3a39354b7546f8607b96f5efdb Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Wed, 16 Jul 2025 04:00:08 -0400 Subject: [PATCH 15/49] Fix unit-test bugs. --- components/core/tests/test-ParserWithUserSchema.cpp | 2 +- components/core/tests/test_schema_files/search_schema.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/tests/test-ParserWithUserSchema.cpp b/components/core/tests/test-ParserWithUserSchema.cpp index 830badea3a..7b018607b0 100644 --- a/components/core/tests/test-ParserWithUserSchema.cpp +++ b/components/core/tests/test-ParserWithUserSchema.cpp @@ -101,7 +101,7 @@ TEST_CASE("Test error for colon missing schema file", "[LALR1Parser][SchemaParse std::string file_path = "../tests/test_schema_files/colon_missing_schema.txt"; REQUIRE_THROWS_WITH( generate_schema_ast(file_path), - "Schema:3:4: error: expected ':','AlphaNumeric' before ' ' token\n" + "Schema:3:4: error: expected '.', ':', 'AlphaNumeric' before ' ' token\n" " int [0-9]+\n" " ^\n" ); diff --git a/components/core/tests/test_schema_files/search_schema.txt b/components/core/tests/test_schema_files/search_schema.txt index f49a6dbfa4..60d0c12f00 100644 --- a/components/core/tests/test_schema_files/search_schema.txt +++ b/components/core/tests/test_schema_files/search_schema.txt @@ -1,5 +1,5 @@ // Delimiters -delimiters: \r\n:,=!;%? +delimiters: \r\n:,=!;%\? // First set of variables timestamp:[0-9]{4}\-[0-9]{2}\-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]{3}){0,1} From da1864629e5356d9930b30859b9a05bbe0c86321 Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Wed, 16 Jul 2025 04:03:11 -0400 Subject: [PATCH 16/49] Fix unit-test typo and spacing. --- components/core/tests/test-ParserWithUserSchema.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/tests/test-ParserWithUserSchema.cpp b/components/core/tests/test-ParserWithUserSchema.cpp index 7b018607b0..7fef6db2ef 100644 --- a/components/core/tests/test-ParserWithUserSchema.cpp +++ b/components/core/tests/test-ParserWithUserSchema.cpp @@ -101,7 +101,7 @@ TEST_CASE("Test error for colon missing schema file", "[LALR1Parser][SchemaParse std::string file_path = "../tests/test_schema_files/colon_missing_schema.txt"; REQUIRE_THROWS_WITH( generate_schema_ast(file_path), - "Schema:3:4: error: expected '.', ':', 'AlphaNumeric' before ' ' token\n" + "Schema:3:4: error: expected '>',':','AlphaNumeric' before ' ' token\n" " int [0-9]+\n" " ^\n" ); From 6dcadca1e9c74a4c2658d8b7f4ad914fb418fbd0 Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Wed, 16 Jul 2025 04:50:35 -0400 Subject: [PATCH 17/49] Update log-surgeon to newest version. --- taskfiles/deps/main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 0aae9d79de..0b5d1d595c 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -235,8 +235,8 @@ tasks: - "-DCMAKE_INSTALL_MESSAGE=LAZY" - "-Dlog_surgeon_BUILD_TESTING=OFF" LIB_NAME: "log_surgeon" - TARBALL_SHA256: "9b723033ed4d6ea1e6831014d1d0921982399cecd076c9418b4dd780e72743ad" - TARBALL_URL: "https://github.com/y-scope/log-surgeon/archive/c954252.tar.gz" + TARBALL_SHA256: "03ee61a03c52cc1de04d5e9f35e2d256b1a6850c063509b5ad89126846ef762d" + TARBALL_URL: "https://github.com/y-scope/log-surgeon/archive/3b2fbcd.tar.gz" microsoft.gsl: internal: true From 08440417daba9ccfa8c9b19a22b72348afea3848 Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Wed, 16 Jul 2025 05:25:45 -0400 Subject: [PATCH 18/49] Remove profiling changes. --- components/core/src/clp/Profiler.hpp | 20 ++----------------- .../core/src/clp/clp/FileCompressor.cpp | 5 +++-- components/core/src/clp/clp/run.cpp | 9 --------- .../make_dictionaries_readable/CMakeLists.txt | 4 ---- 4 files changed, 5 insertions(+), 33 deletions(-) diff --git a/components/core/src/clp/Profiler.hpp b/components/core/src/clp/Profiler.hpp index a355df17a4..3e25180f27 100644 --- a/components/core/src/clp/Profiler.hpp +++ b/components/core/src/clp/Profiler.hpp @@ -41,19 +41,11 @@ class Profiler { // Types enum class ContinuousMeasurementIndex : size_t { Compression = 0, + ParseLogFile, Search, Length }; enum class FragmentedMeasurementIndex : size_t { - Watch, - ParseLogFile, - Parse, - GetNext, - ProcessState, - ScanWhileLoop, - TokenCreation, - ProcessCharWatch, - ProcessChar, Length }; @@ -62,20 +54,12 @@ class Profiler { static constexpr auto cContinuousMeasurementEnabled = []() { std::array enabled{}; enabled[enum_to_underlying_type(ContinuousMeasurementIndex::Compression)] = false; + enabled[enum_to_underlying_type(ContinuousMeasurementIndex::ParseLogFile)] = false; enabled[enum_to_underlying_type(ContinuousMeasurementIndex::Search)] = false; return enabled; }(); static constexpr auto cFragmentedMeasurementEnabled = []() { std::array enabled{}; - enabled[enum_to_underlying_type(FragmentedMeasurementIndex::ParseLogFile)] = false; - enabled[enum_to_underlying_type(FragmentedMeasurementIndex::Parse)] = false; - enabled[enum_to_underlying_type(FragmentedMeasurementIndex::GetNext)] = false; - enabled[enum_to_underlying_type(FragmentedMeasurementIndex::ProcessState)] = false; - enabled[enum_to_underlying_type(FragmentedMeasurementIndex::ScanWhileLoop)] = false; - enabled[enum_to_underlying_type(FragmentedMeasurementIndex::Watch)] = true; - enabled[enum_to_underlying_type(FragmentedMeasurementIndex::TokenCreation)] = true; - enabled[enum_to_underlying_type(FragmentedMeasurementIndex::ProcessCharWatch)] = false; - enabled[enum_to_underlying_type(FragmentedMeasurementIndex::ProcessChar)] = false; return enabled; }(); diff --git a/components/core/src/clp/clp/FileCompressor.cpp b/components/core/src/clp/clp/FileCompressor.cpp index 544192f7bf..31badda437 100644 --- a/components/core/src/clp/clp/FileCompressor.cpp +++ b/components/core/src/clp/clp/FileCompressor.cpp @@ -119,7 +119,7 @@ bool FileCompressor::compress_file( std::string file_name = std::filesystem::canonical(file_to_compress.get_path()).string(); PROFILER_SPDLOG_INFO("Start parsing {}", file_name) - Profiler::start_fragmented_measurement(); + Profiler::start_continuous_measurement(); m_file_reader.open(file_to_compress.get_path()); @@ -186,7 +186,8 @@ bool FileCompressor::compress_file( m_file_reader.close(); - Profiler::stop_fragmented_measurement(); + Profiler::stop_continuous_measurement(); + LOG_CONTINUOUS_MEASUREMENT(Profiler::ContinuousMeasurementIndex::ParseLogFile) PROFILER_SPDLOG_INFO("Done parsing {}", file_name) return succeeded; diff --git a/components/core/src/clp/clp/run.cpp b/components/core/src/clp/clp/run.cpp index 06f4f8fb3e..b70bf45ff4 100644 --- a/components/core/src/clp/clp/run.cpp +++ b/components/core/src/clp/clp/run.cpp @@ -152,15 +152,6 @@ int run(int argc, char const* argv[]) { Profiler::stop_continuous_measurement(); LOG_CONTINUOUS_MEASUREMENT(Profiler::ContinuousMeasurementIndex::Compression) - LOG_FRAGMENTED_MEASUREMENT(Profiler::FragmentedMeasurementIndex::Watch) - LOG_FRAGMENTED_MEASUREMENT(Profiler::FragmentedMeasurementIndex::Parse) - LOG_FRAGMENTED_MEASUREMENT(Profiler::FragmentedMeasurementIndex::GetNext) - LOG_FRAGMENTED_MEASUREMENT(Profiler::FragmentedMeasurementIndex::ProcessState) - LOG_FRAGMENTED_MEASUREMENT(Profiler::FragmentedMeasurementIndex::ScanWhileLoop) - LOG_FRAGMENTED_MEASUREMENT(Profiler::FragmentedMeasurementIndex::TokenCreation) - LOG_FRAGMENTED_MEASUREMENT(Profiler::FragmentedMeasurementIndex::ParseLogFile) - LOG_FRAGMENTED_MEASUREMENT(Profiler::FragmentedMeasurementIndex::ProcessCharWatch) - LOG_FRAGMENTED_MEASUREMENT(Profiler::FragmentedMeasurementIndex::ProcessChar) return 0; } diff --git a/components/core/src/clp/make_dictionaries_readable/CMakeLists.txt b/components/core/src/clp/make_dictionaries_readable/CMakeLists.txt index b628072d66..65bf33e03e 100644 --- a/components/core/src/clp/make_dictionaries_readable/CMakeLists.txt +++ b/components/core/src/clp/make_dictionaries_readable/CMakeLists.txt @@ -17,15 +17,11 @@ set( ../LogTypeDictionaryReader.hpp ../ParsedMessage.cpp ../ParsedMessage.hpp - ../Profiler.hpp - ../Profiler.cpp ../ReaderInterface.cpp ../ReaderInterface.hpp ../ReadOnlyMemoryMappedFile.cpp ../ReadOnlyMemoryMappedFile.hpp ../spdlog_with_specializations.hpp - ../Stopwatch.cpp - ../Stopwatch.hpp ../streaming_compression/Decompressor.hpp ../streaming_compression/passthrough/Decompressor.cpp ../streaming_compression/passthrough/Decompressor.hpp From 3c20be7c06ca29f3834347140784eca5c67aa650 Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Wed, 16 Jul 2025 06:06:08 -0400 Subject: [PATCH 19/49] Test disabling macos-14. --- .github/workflows/clp-core-build-macos.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 51eb6985d0..99a24e62a8 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -46,7 +46,6 @@ jobs: matrix: os: - "macos-13" - - "macos-14" - "macos-15" use_shared_libs: - true From e8d5fbb959171a7f160c53d3f3cc327fe507dbba Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Wed, 16 Jul 2025 06:11:54 -0400 Subject: [PATCH 20/49] Test disabling macos-13. --- .github/workflows/clp-core-build-macos.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 99a24e62a8..ebf567cdb7 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -45,7 +45,6 @@ jobs: strategy: matrix: os: - - "macos-13" - "macos-15" use_shared_libs: - true From c8edd586bbd071c14b0412c18ba218c99aac7cf0 Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Wed, 16 Jul 2025 06:46:47 -0400 Subject: [PATCH 21/49] Readd macos-13 and macos-14 to the CI. --- .github/workflows/clp-core-build-macos.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index ebf567cdb7..51eb6985d0 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -45,6 +45,8 @@ jobs: strategy: matrix: os: + - "macos-13" + - "macos-14" - "macos-15" use_shared_libs: - true From c0f86e9107548aad34ebb21180d4ff6230ed229f Mon Sep 17 00:00:00 2001 From: davidlion Date: Wed, 16 Jul 2025 10:23:41 -0400 Subject: [PATCH 22/49] Bump log-surgeon version. --- taskfiles/deps/main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 0b5d1d595c..e4cbd258c6 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -235,8 +235,8 @@ tasks: - "-DCMAKE_INSTALL_MESSAGE=LAZY" - "-Dlog_surgeon_BUILD_TESTING=OFF" LIB_NAME: "log_surgeon" - TARBALL_SHA256: "03ee61a03c52cc1de04d5e9f35e2d256b1a6850c063509b5ad89126846ef762d" - TARBALL_URL: "https://github.com/y-scope/log-surgeon/archive/3b2fbcd.tar.gz" + TARBALL_SHA256: "d91a7469db437162d518eb0f9175268084b478b5003163d0f6233abbbce9b0f8" + TARBALL_URL: "https://github.com/y-scope/log-surgeon/archive/dfd5c79.tar.gz" microsoft.gsl: internal: true From f631545bb6a9d7adacc74ac87f2d3768e8dacb53 Mon Sep 17 00:00:00 2001 From: davidlion Date: Wed, 16 Jul 2025 10:40:54 -0400 Subject: [PATCH 23/49] Add spacing to schemas.txt. --- components/core/config/schemas.txt | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index 1dd3811db6..1d3a36c3f1 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -1,51 +1,73 @@ -// Timestamps (using the `timestamp` keyword) +// Timestamps + // E.g. 2015-01-31T15:50:45.392 timestamp:\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}.\d{3} + // E.g. 2015-01-31T15:50:45,392 timestamp:\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2},\d{3} + // E.g. [2015-01-31T15:50:45 timestamp:\[\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2} + // E.g. [20170106-16:56:41] timestamp:\[\d{4}\d{2}\d{2}\-\d{2}:\d{2}:\d{2}\] + // E.g. 2015-01-31 15:50:45,392 // E.g. INFO [main] 2015-01-31 15:50:45,085 timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2},\d{3} + // E.g. 2015-01-31 15:50:45.392 timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}.\d{3} + // E.g. [2015-01-31 15:50:45,085] timestamp:\[\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2},\d{3}\] + // E.g. 2015-01-31 15:50:45 // E.g. Started POST /api/v3/internal/allowed for 127.0.0.1 at 2017-06-18 00:20:44 // E.g. update-alternatives 2015-01-31 15:50:45 timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2} + // E.g. Start-Date: 2015-01-31 15:50:45 timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2} + // E.g. 2015/01/31 15:50:45 timestamp:\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} + // E.g. 15/01/31 15:50:45 timestamp:\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} + // E.g. 150131 9:50:45 timestamp:\d{2}\d{2}\d{2} [ 0-9]{2}:\d{2}:\d{2} + // E.g. 01 Jan 2016 15:50:17,085 timestamp:\d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2},\d{3} + // E.g. Jan 01, 2016 3:50:17 PM timestamp:[A-Z][a-z]{2} \d{2}, \d{4} [ 0-9]{2}:\d{2}:\d{2} [AP]M + // E.g. January 31, 2015 15:50 timestamp:[A-Z][a-z]+ \d{2}, \d{4} \d{2}:\d{2} + // E.g. E [31/Jan/2015:15:50:45 // E.g. localhost - - [01/Jan/2016:15:50:17 // E.g. 192.168.4.5 - - [01/Jan/2016:15:50:17 timestamp:\[\d{2}/[A-Z][a-z]{2}/\d{4}:\d{2}:\d{2}:\d{2} + // E.g. 192.168.4.5 - - [01/01/2016:15:50:17 timestamp:\[\d{2}/\d{2}/\d{4}:\d{2}:\d{2}:\d{2} + // E.g. ERROR: apport (pid 4557) Sun Jan 1 15:50:45 2015 timestamp:[A-Z][a-z]{2} [A-Z][a-z]{2} [ 0-9]{2} \d{2}:\d{2}:\d{2} \d{4} + // E.g. <<<2016-11-10 03:02:29:936 timestamp:\<\<\<\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}:\d{3} + // E.g. Jan 21 11:56:42 timestamp:[A-Z][a-z]{2} \d{2} \d{2}:\d{2}:\d{2} + // E.g. 01-21 11:56:42.392 timestamp:\d{2}\-\d{2} \d{2}:\d{2}:\d{2}.\d{3} + // E.g. 2016-05-08 11:34:04.083464 timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}.\d{6} From bb23a83f8d534ff91423000b57abfd9c8fbcee7f Mon Sep 17 00:00:00 2001 From: davidlion Date: Wed, 16 Jul 2025 10:45:37 -0400 Subject: [PATCH 24/49] Drop unused parameter from load_lexer_from_file. --- components/core/src/clp/Utils.cpp | 8 +------- components/core/src/clp/Utils.hpp | 2 -- components/core/src/clp/clg/clg.cpp | 8 ++++---- components/core/src/clp/clo/clo.cpp | 4 ++-- components/core/tests/test-Grep.cpp | 4 ++-- components/core/tests/test-ParserWithUserSchema.cpp | 4 ++-- 6 files changed, 11 insertions(+), 19 deletions(-) diff --git a/components/core/src/clp/Utils.cpp b/components/core/src/clp/Utils.cpp index 8e65d6c486..818f24f659 100644 --- a/components/core/src/clp/Utils.cpp +++ b/components/core/src/clp/Utils.cpp @@ -123,7 +123,6 @@ ErrorCode read_list_of_paths(string const& list_path, vector& paths) { // lexer.m_symbol_id, contains_delimiter error, and add_rule logic. void load_lexer_from_file( std::string const& schema_file_path, - bool reverse, log_surgeon::lexers::ByteLexer& lexer ) { std::unique_ptr schema_ast @@ -242,11 +241,6 @@ void load_lexer_from_file( } lexer.add_rule(lexer.m_symbol_id[rule->m_name], std::move(rule->m_regex_ptr)); } - if (reverse) { - // TODO: This isn't used anymore for the new search, supporting it here is a waste of time - // lexer.generate_reverse(); - } else { - lexer.generate(); - } + lexer.generate(); } } // namespace clp diff --git a/components/core/src/clp/Utils.hpp b/components/core/src/clp/Utils.hpp index 3238e551b8..41728ee558 100644 --- a/components/core/src/clp/Utils.hpp +++ b/components/core/src/clp/Utils.hpp @@ -47,12 +47,10 @@ ErrorCode read_list_of_paths(std::string const& list_path, std::vectorsecond; - load_lexer_from_file(schema_file_path, false, *forward_lexer_ptr); + load_lexer_from_file(schema_file_path, *forward_lexer_ptr); // Create reverse lexer insert_result = reverse_lexer_map.emplace(buf, log_surgeon::lexers::ByteLexer()); reverse_lexer_ptr = &insert_result.first->second; - load_lexer_from_file(schema_file_path, true, *reverse_lexer_ptr); + load_lexer_from_file(schema_file_path, *reverse_lexer_ptr); } else { // load the lexers if they already exist forward_lexer_ptr = &forward_lexer_map_it->second; @@ -616,11 +616,11 @@ int main(int argc, char const* argv[]) { } else { // Create forward lexer forward_lexer_ptr = &one_time_use_forward_lexer; - load_lexer_from_file(schema_file_path, false, one_time_use_forward_lexer); + load_lexer_from_file(schema_file_path, one_time_use_forward_lexer); // Create reverse lexer reverse_lexer_ptr = &one_time_use_reverse_lexer; - load_lexer_from_file(schema_file_path, false, one_time_use_reverse_lexer); + load_lexer_from_file(schema_file_path, one_time_use_reverse_lexer); } } diff --git a/components/core/src/clp/clo/clo.cpp b/components/core/src/clp/clo/clo.cpp index a675f9fe6a..97b0d392bb 100644 --- a/components/core/src/clp/clo/clo.cpp +++ b/components/core/src/clp/clo/clo.cpp @@ -482,11 +482,11 @@ static bool search_archive( use_heuristic = false; // Create forward lexer forward_lexer.reset(new log_surgeon::lexers::ByteLexer()); - load_lexer_from_file(schema_file_path.string(), false, *forward_lexer); + load_lexer_from_file(schema_file_path.string(), *forward_lexer); // Create reverse lexer reverse_lexer.reset(new log_surgeon::lexers::ByteLexer()); - load_lexer_from_file(schema_file_path.string(), true, *reverse_lexer); + load_lexer_from_file(schema_file_path.string(), *reverse_lexer); } Archive archive_reader; diff --git a/components/core/tests/test-Grep.cpp b/components/core/tests/test-Grep.cpp index f0bdb21dc3..de59bc6bbc 100644 --- a/components/core/tests/test-Grep.cpp +++ b/components/core/tests/test-Grep.cpp @@ -18,9 +18,9 @@ using std::string; TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var]") { ByteLexer forward_lexer; - load_lexer_from_file("../tests/test_schema_files/search_schema.txt", false, forward_lexer); + load_lexer_from_file("../tests/test_schema_files/search_schema.txt", forward_lexer); ByteLexer reverse_lexer; - load_lexer_from_file("../tests/test_schema_files/search_schema.txt", true, reverse_lexer); + load_lexer_from_file("../tests/test_schema_files/search_schema.txt", reverse_lexer); string str; size_t begin_pos; diff --git a/components/core/tests/test-ParserWithUserSchema.cpp b/components/core/tests/test-ParserWithUserSchema.cpp index 7fef6db2ef..83674f38eb 100644 --- a/components/core/tests/test-ParserWithUserSchema.cpp +++ b/components/core/tests/test-ParserWithUserSchema.cpp @@ -159,7 +159,7 @@ TEST_CASE("Test forward lexer", "[Search]") { ByteLexer forward_lexer; std::string schema_file_name = "../tests/test_schema_files/search_schema.txt"; std::string schema_file_path = boost::filesystem::weakly_canonical(schema_file_name).string(); - load_lexer_from_file(schema_file_path, false, forward_lexer); + load_lexer_from_file(schema_file_path, forward_lexer); FileReader file_reader{"../tests/test_search_queries/easy.txt"}; LogSurgeonReader reader_wrapper(file_reader); log_surgeon::ParserInputBuffer parser_input_buffer; @@ -184,7 +184,7 @@ TEST_CASE("Test reverse lexer", "[Search]") { ByteLexer reverse_lexer; std::string schema_file_name = "../tests/test_schema_files/search_schema.txt"; std::string schema_file_path = boost::filesystem::weakly_canonical(schema_file_name).string(); - load_lexer_from_file(schema_file_path, false, reverse_lexer); + load_lexer_from_file(schema_file_path, reverse_lexer); FileReader file_reader{"../tests/test_search_queries/easy.txt"}; LogSurgeonReader reader_wrapper(file_reader); log_surgeon::ParserInputBuffer parser_input_buffer; From ea777c3e1dfcf129426433caa11ff50c551b709e Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Wed, 16 Jul 2025 11:28:08 -0400 Subject: [PATCH 25/49] Remove a missed benchmark change. --- components/core/src/clp/Profiler.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/core/src/clp/Profiler.hpp b/components/core/src/clp/Profiler.hpp index 3e25180f27..f93dec0708 100644 --- a/components/core/src/clp/Profiler.hpp +++ b/components/core/src/clp/Profiler.hpp @@ -53,9 +53,9 @@ class Profiler { // NOTE: We use lambdas so that we can programmatically initialize the constexpr array static constexpr auto cContinuousMeasurementEnabled = []() { std::array enabled{}; - enabled[enum_to_underlying_type(ContinuousMeasurementIndex::Compression)] = false; - enabled[enum_to_underlying_type(ContinuousMeasurementIndex::ParseLogFile)] = false; - enabled[enum_to_underlying_type(ContinuousMeasurementIndex::Search)] = false; + enabled[enum_to_underlying_type(ContinuousMeasurementIndex::Compression)] = true; + enabled[enum_to_underlying_type(ContinuousMeasurementIndex::ParseLogFile)] = true; + enabled[enum_to_underlying_type(ContinuousMeasurementIndex::Search)] = true; return enabled; }(); static constexpr auto cFragmentedMeasurementEnabled = []() { From 1d63a20396f281c128a5b806ac468ce5c85aeee8 Mon Sep 17 00:00:00 2001 From: davidlion Date: Wed, 16 Jul 2025 11:37:58 -0400 Subject: [PATCH 26/49] Format fix. --- components/core/src/clp/Utils.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/components/core/src/clp/Utils.cpp b/components/core/src/clp/Utils.cpp index 818f24f659..f52cb87cdf 100644 --- a/components/core/src/clp/Utils.cpp +++ b/components/core/src/clp/Utils.cpp @@ -121,10 +121,8 @@ ErrorCode read_list_of_paths(string const& list_path, vector& paths) { // TODO: duplicates code in log_surgeon/parser.tpp, should implement a // SearchParser in log_surgeon instead and use it here. Specifically, initialization of // lexer.m_symbol_id, contains_delimiter error, and add_rule logic. -void load_lexer_from_file( - std::string const& schema_file_path, - log_surgeon::lexers::ByteLexer& lexer -) { +void +load_lexer_from_file(std::string const& schema_file_path, log_surgeon::lexers::ByteLexer& lexer) { std::unique_ptr schema_ast = log_surgeon::SchemaParser::try_schema_file(schema_file_path); if (!lexer.m_symbol_id.empty()) { From 95522728a5a6d334bcc1e0524404009c992d3c81 Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Wed, 16 Jul 2025 12:45:27 -0400 Subject: [PATCH 27/49] Remove reverse lexer; Rename forward lexer to just lexer. --- components/core/src/clp/Grep.cpp | 39 ++++--------- components/core/src/clp/Grep.hpp | 12 ++-- components/core/src/clp/Utils.hpp | 4 +- components/core/src/clp/clg/clg.cpp | 49 +++++----------- components/core/src/clp/clo/clo.cpp | 14 ++--- components/core/tests/test-Grep.cpp | 57 +++++++------------ .../core/tests/test-ParserWithUserSchema.cpp | 39 +++---------- 7 files changed, 62 insertions(+), 152 deletions(-) diff --git a/components/core/src/clp/Grep.cpp b/components/core/src/clp/Grep.cpp index f34aecc0d6..ae03d9dfb3 100644 --- a/components/core/src/clp/Grep.cpp +++ b/components/core/src/clp/Grep.cpp @@ -502,8 +502,7 @@ std::optional Grep::process_raw_query( epochtime_t search_begin_ts, epochtime_t search_end_ts, bool ignore_case, - log_surgeon::lexers::ByteLexer& forward_lexer, - log_surgeon::lexers::ByteLexer& reverse_lexer, + log_surgeon::lexers::ByteLexer& lexer, bool use_heuristic ) { // Add prefix and suffix '*' to make the search a sub-string match @@ -546,8 +545,7 @@ std::optional Grep::process_raw_query( begin_pos, end_pos, is_var, - forward_lexer, - reverse_lexer + lexer )) { query_tokens.emplace_back(search_string_for_sub_queries, begin_pos, end_pos, is_var); @@ -752,8 +750,7 @@ bool Grep::get_bounds_of_next_potential_var( size_t& begin_pos, size_t& end_pos, bool& is_var, - log_surgeon::lexers::ByteLexer& forward_lexer, - log_surgeon::lexers::ByteLexer& reverse_lexer + log_surgeon::lexers::ByteLexer& lexer ) { size_t const value_length = value.length(); if (end_pos >= value_length) { @@ -774,7 +771,7 @@ bool Grep::get_bounds_of_next_potential_var( if (is_escaped) { is_escaped = false; - if (false == forward_lexer.is_delimiter(c)) { + if (false == lexer.is_delimiter(c)) { // Found escaped non-delimiter, so reverse the index to retain the escape // character --begin_pos; @@ -788,7 +785,7 @@ bool Grep::get_bounds_of_next_potential_var( contains_wildcard = true; break; } - if (false == forward_lexer.is_delimiter(c)) { + if (false == lexer.is_delimiter(c)) { break; } } @@ -803,7 +800,7 @@ bool Grep::get_bounds_of_next_potential_var( if (is_escaped) { is_escaped = false; - if (forward_lexer.is_delimiter(c)) { + if (lexer.is_delimiter(c)) { // Found escaped delimiter, so reverse the index to retain the escape character --end_pos; break; @@ -814,7 +811,7 @@ bool Grep::get_bounds_of_next_potential_var( } else { if (is_wildcard(c)) { contains_wildcard = true; - } else if (forward_lexer.is_delimiter(c)) { + } else if (lexer.is_delimiter(c)) { // Found delimiter that's not also a wildcard break; } @@ -832,7 +829,7 @@ bool Grep::get_bounds_of_next_potential_var( } } SearchToken search_token; - if (has_wildcard_in_middle || (has_prefix_wildcard && has_suffix_wildcard)) { + if (has_wildcard_in_middle || has_prefix_wildcard) { // DO NOTHING } else { StringReader string_reader; @@ -844,29 +841,17 @@ bool Grep::get_bounds_of_next_potential_var( // string, should be improved when adding a SearchParser to log_surgeon string_reader.open(value.substr(begin_pos, end_pos - begin_pos - 1)); parser_input_buffer.read_if_safe(reader_wrapper); - forward_lexer.reset(); - forward_lexer.scan_with_wildcard( + lexer.reset(); + lexer.scan_with_wildcard( parser_input_buffer, value[end_pos - 1], search_token ); - } else if (has_prefix_wildcard) { // *text - std::string value_reverse - = value.substr(begin_pos + 1, end_pos - begin_pos - 1); - std::reverse(value_reverse.begin(), value_reverse.end()); - string_reader.open(value_reverse); - parser_input_buffer.read_if_safe(reader_wrapper); - reverse_lexer.reset(); - reverse_lexer.scan_with_wildcard( - parser_input_buffer, - value[begin_pos], - search_token - ); } else { // no wildcards string_reader.open(value.substr(begin_pos, end_pos - begin_pos)); parser_input_buffer.read_if_safe(reader_wrapper); - forward_lexer.reset(); - auto [err, token] = forward_lexer.scan(parser_input_buffer); + lexer.reset(); + auto [err, token] = lexer.scan(parser_input_buffer); if (log_surgeon::ErrorCode::Success != err) { return false; } diff --git a/components/core/src/clp/Grep.hpp b/components/core/src/clp/Grep.hpp index f520af2120..d4e965c7ee 100644 --- a/components/core/src/clp/Grep.hpp +++ b/components/core/src/clp/Grep.hpp @@ -37,8 +37,7 @@ class Grep { * @param search_begin_ts * @param search_end_ts * @param ignore_case - * @param forward_lexer DFA for determining if input is in the schema - * @param reverse_lexer DFA for determining if reverse of input is in the schema + * @param lexer DFA for determining if input is in the schema * @param use_heuristic * @return Query if it may match a message, std::nullopt otherwise */ @@ -48,8 +47,7 @@ class Grep { epochtime_t search_begin_ts, epochtime_t search_end_ts, bool ignore_case, - log_surgeon::lexers::ByteLexer& forward_lexer, - log_surgeon::lexers::ByteLexer& reverse_lexer, + log_surgeon::lexers::ByteLexer& lexer, bool use_heuristic ); @@ -76,8 +74,7 @@ class Grep { * @param begin_pos Begin position of last token, changes to begin position of next token * @param end_pos End position of last token, changes to end position of next token * @param is_var Whether the token is definitely a variable - * @param forward_lexer DFA for determining if input is in the schema - * @param reverse_lexer DFA for determining if reverse of input is in the schema + * @param lexer DFA for determining if input is in the schema * @return true if another potential variable was found, false otherwise */ static bool get_bounds_of_next_potential_var( @@ -85,8 +82,7 @@ class Grep { size_t& begin_pos, size_t& end_pos, bool& is_var, - log_surgeon::lexers::ByteLexer& forward_lexer, - log_surgeon::lexers::ByteLexer& reverse_lexer + log_surgeon::lexers::ByteLexer& lexer ); /** * Marks which sub-queries in each query are relevant to the given file diff --git a/components/core/src/clp/Utils.hpp b/components/core/src/clp/Utils.hpp index 41728ee558..9ccae55bf2 100644 --- a/components/core/src/clp/Utils.hpp +++ b/components/core/src/clp/Utils.hpp @@ -47,11 +47,11 @@ ErrorCode read_list_of_paths(std::string const& list_path, std::vector const& search_strings, CommandLineArguments& command_line_args, Archive& archive, - log_surgeon::lexers::ByteLexer& forward_lexer, - log_surgeon::lexers::ByteLexer& reverse_lexer, + log_surgeon::lexers::ByteLexer& lexer, bool use_heuristic ) { ErrorCode error_code; @@ -225,8 +224,7 @@ static bool search( search_begin_ts, search_end_ts, command_line_args.ignore_case(), - forward_lexer, - reverse_lexer, + lexer, use_heuristic ); if (query_processing_result.has_value()) { @@ -545,12 +543,9 @@ int main(int argc, char const* argv[]) { // TODO: if performance is too slow, can make this more efficient by only diffing files with the // same checksum uint32_t const max_map_schema_length = 100'000; - std::map forward_lexer_map; - std::map reverse_lexer_map; - log_surgeon::lexers::ByteLexer one_time_use_forward_lexer; - log_surgeon::lexers::ByteLexer one_time_use_reverse_lexer; - log_surgeon::lexers::ByteLexer* forward_lexer_ptr; - log_surgeon::lexers::ByteLexer* reverse_lexer_ptr; + std::map lexer_map; + log_surgeon::lexers::ByteLexer one_time_use_lexer; + log_surgeon::lexers::ByteLexer* lexer_ptr; string archive_id; Archive archive_reader; @@ -592,35 +587,20 @@ int main(int argc, char const* argv[]) { size_t num_bytes_read; file_reader.read(buf, max_map_schema_length, num_bytes_read); if (num_bytes_read < max_map_schema_length) { - auto forward_lexer_map_it = forward_lexer_map.find(buf); - auto reverse_lexer_map_it = reverse_lexer_map.find(buf); + auto lexer_map_it = lexer_map.find(buf); // if there is a chance there might be a difference make a new lexer as it's pretty // fast to create - if (forward_lexer_map_it == forward_lexer_map.end()) { - // Create forward lexer + if (lexer_map_it == lexer_map.end()) { auto insert_result - = forward_lexer_map.emplace(buf, log_surgeon::lexers::ByteLexer()); - forward_lexer_ptr = &insert_result.first->second; - load_lexer_from_file(schema_file_path, *forward_lexer_ptr); - - // Create reverse lexer - insert_result - = reverse_lexer_map.emplace(buf, log_surgeon::lexers::ByteLexer()); - reverse_lexer_ptr = &insert_result.first->second; - load_lexer_from_file(schema_file_path, *reverse_lexer_ptr); + = lexer_map.emplace(buf, log_surgeon::lexers::ByteLexer()); + lexer_ptr = &insert_result.first->second; + load_lexer_from_file(schema_file_path, *lexer_ptr); } else { - // load the lexers if they already exist - forward_lexer_ptr = &forward_lexer_map_it->second; - reverse_lexer_ptr = &reverse_lexer_map_it->second; + lexer_ptr = &lexer_map_it->second; } } else { - // Create forward lexer - forward_lexer_ptr = &one_time_use_forward_lexer; - load_lexer_from_file(schema_file_path, one_time_use_forward_lexer); - - // Create reverse lexer - reverse_lexer_ptr = &one_time_use_reverse_lexer; - load_lexer_from_file(schema_file_path, one_time_use_reverse_lexer); + lexer_ptr = &one_time_use_lexer; + load_lexer_from_file(schema_file_path, one_time_use_lexer); } } @@ -628,8 +608,7 @@ int main(int argc, char const* argv[]) { if (!search(search_strings, command_line_args, archive_reader, - *forward_lexer_ptr, - *reverse_lexer_ptr, + *lexer_ptr, use_heuristic)) { return -1; diff --git a/components/core/src/clp/clo/clo.cpp b/components/core/src/clp/clo/clo.cpp index 97b0d392bb..a8118cdbb2 100644 --- a/components/core/src/clp/clo/clo.cpp +++ b/components/core/src/clp/clo/clo.cpp @@ -476,17 +476,12 @@ static bool search_archive( // Load lexers from schema file if it exists auto schema_file_path = archive_path / clp::streaming_archive::cSchemaFileName; - unique_ptr forward_lexer, reverse_lexer; + unique_ptr lexer; bool use_heuristic = true; if (std::filesystem::exists(schema_file_path)) { use_heuristic = false; - // Create forward lexer - forward_lexer.reset(new log_surgeon::lexers::ByteLexer()); - load_lexer_from_file(schema_file_path.string(), *forward_lexer); - - // Create reverse lexer - reverse_lexer.reset(new log_surgeon::lexers::ByteLexer()); - load_lexer_from_file(schema_file_path.string(), *reverse_lexer); + lexer.reset(new log_surgeon::lexers::ByteLexer()); + load_lexer_from_file(schema_file_path.string(), *lexer); } Archive archive_reader; @@ -502,8 +497,7 @@ static bool search_archive( search_begin_ts, search_end_ts, command_line_args.ignore_case(), - *forward_lexer, - *reverse_lexer, + *lexer, use_heuristic ); if (false == query_processing_result.has_value()) { diff --git a/components/core/tests/test-Grep.cpp b/components/core/tests/test-Grep.cpp index de59bc6bbc..69eb27aee6 100644 --- a/components/core/tests/test-Grep.cpp +++ b/components/core/tests/test-Grep.cpp @@ -17,10 +17,8 @@ using log_surgeon::SchemaVarAST; using std::string; TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var]") { - ByteLexer forward_lexer; - load_lexer_from_file("../tests/test_schema_files/search_schema.txt", forward_lexer); - ByteLexer reverse_lexer; - load_lexer_from_file("../tests/test_schema_files/search_schema.txt", reverse_lexer); + ByteLexer lexer; + load_lexer_from_file("../tests/test_schema_files/search_schema.txt", lexer); string str; size_t begin_pos; @@ -36,8 +34,7 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos, end_pos, is_var, - forward_lexer, - reverse_lexer + lexer ) == false); @@ -50,8 +47,7 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos, end_pos, is_var, - forward_lexer, - reverse_lexer + lexer ) == false); @@ -64,8 +60,7 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos, end_pos, is_var, - forward_lexer, - reverse_lexer + lexer ) == false); @@ -79,8 +74,7 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos, end_pos, is_var, - forward_lexer, - reverse_lexer + lexer ) == true); REQUIRE("95" == str.substr(begin_pos, end_pos - begin_pos)); @@ -91,8 +85,7 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos, end_pos, is_var, - forward_lexer, - reverse_lexer + lexer ) == true); REQUIRE("ad" == str.substr(begin_pos, end_pos - begin_pos)); @@ -103,8 +96,7 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos, end_pos, is_var, - forward_lexer, - reverse_lexer + lexer ) == true); REQUIRE("ff" == str.substr(begin_pos, end_pos - begin_pos)); @@ -115,8 +107,7 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos, end_pos, is_var, - forward_lexer, - reverse_lexer + lexer ) == true); REQUIRE("95" == str.substr(begin_pos, end_pos - begin_pos)); @@ -127,8 +118,7 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos, end_pos, is_var, - forward_lexer, - reverse_lexer + lexer ) == true); REQUIRE("24" == str.substr(begin_pos, end_pos - begin_pos)); @@ -139,8 +129,7 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos, end_pos, is_var, - forward_lexer, - reverse_lexer + lexer ) == true); REQUIRE("0d" == str.substr(begin_pos, end_pos - begin_pos)); @@ -151,8 +140,7 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos, end_pos, is_var, - forward_lexer, - reverse_lexer + lexer ) == true); REQUIRE("ff" == str.substr(begin_pos, end_pos - begin_pos)); @@ -167,8 +155,7 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos, end_pos, is_var, - forward_lexer, - reverse_lexer + lexer ) == false); REQUIRE(str.length() == begin_pos); @@ -183,8 +170,7 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos, end_pos, is_var, - forward_lexer, - reverse_lexer + lexer ) == true); REQUIRE(str.substr(begin_pos, end_pos - begin_pos) == "1\\*x"); @@ -196,8 +182,7 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos, end_pos, is_var, - forward_lexer, - reverse_lexer + lexer ) == true); REQUIRE(str.substr(begin_pos, end_pos - begin_pos) == "abc*123"); @@ -209,8 +194,7 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos, end_pos, is_var, - forward_lexer, - reverse_lexer + lexer ) == true); REQUIRE(str.substr(begin_pos, end_pos - begin_pos) == "1.2"); @@ -221,8 +205,7 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos, end_pos, is_var, - forward_lexer, - reverse_lexer + lexer ) == true); REQUIRE(str.substr(begin_pos, end_pos - begin_pos) == "+394/-"); @@ -233,8 +216,7 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos, end_pos, is_var, - forward_lexer, - reverse_lexer + lexer ) == true); REQUIRE(str.substr(begin_pos, end_pos - begin_pos) == "-*abc-"); @@ -245,8 +227,7 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos, end_pos, is_var, - forward_lexer, - reverse_lexer + lexer ) == false); } diff --git a/components/core/tests/test-ParserWithUserSchema.cpp b/components/core/tests/test-ParserWithUserSchema.cpp index 83674f38eb..844c7835a9 100644 --- a/components/core/tests/test-ParserWithUserSchema.cpp +++ b/components/core/tests/test-ParserWithUserSchema.cpp @@ -155,51 +155,26 @@ TEST_CASE("Test creating log parser without delimiters", "[LALR1Parser][LogParse // "Specified schema file does not exist."); //} -TEST_CASE("Test forward lexer", "[Search]") { - ByteLexer forward_lexer; +TEST_CASE("Test lexer", "[Search]") { + ByteLexer lexer; std::string schema_file_name = "../tests/test_schema_files/search_schema.txt"; std::string schema_file_path = boost::filesystem::weakly_canonical(schema_file_name).string(); - load_lexer_from_file(schema_file_path, forward_lexer); + load_lexer_from_file(schema_file_path, lexer); FileReader file_reader{"../tests/test_search_queries/easy.txt"}; LogSurgeonReader reader_wrapper(file_reader); log_surgeon::ParserInputBuffer parser_input_buffer; parser_input_buffer.read_if_safe(reader_wrapper); - forward_lexer.reset(); - auto [error_code, opt_token] = forward_lexer.scan(parser_input_buffer); + lexer.reset(); + auto [error_code, opt_token] = lexer.scan(parser_input_buffer); REQUIRE(error_code == log_surgeon::ErrorCode::Success); Token token{opt_token.value()}; while (token.m_type_ids_ptr->at(0) != static_cast(log_surgeon::SymbolId::TokenEnd)) { SPDLOG_INFO("token:" + token.to_string() + "\n"); SPDLOG_INFO( "token.m_type_ids->back():" - + forward_lexer.m_id_symbol[token.m_type_ids_ptr->back()] + "\n" + + lexer.m_id_symbol[token.m_type_ids_ptr->back()] + "\n" ); - auto [error_code, opt_token] = forward_lexer.scan(parser_input_buffer); - REQUIRE(error_code == log_surgeon::ErrorCode::Success); - token = opt_token.value(); - } -} - -TEST_CASE("Test reverse lexer", "[Search]") { - ByteLexer reverse_lexer; - std::string schema_file_name = "../tests/test_schema_files/search_schema.txt"; - std::string schema_file_path = boost::filesystem::weakly_canonical(schema_file_name).string(); - load_lexer_from_file(schema_file_path, reverse_lexer); - FileReader file_reader{"../tests/test_search_queries/easy.txt"}; - LogSurgeonReader reader_wrapper(file_reader); - log_surgeon::ParserInputBuffer parser_input_buffer; - parser_input_buffer.read_if_safe(reader_wrapper); - reverse_lexer.reset(); - auto [error_code, opt_token] = reverse_lexer.scan(parser_input_buffer); - REQUIRE(error_code == log_surgeon::ErrorCode::Success); - Token token{opt_token.value()}; - while (token.m_type_ids_ptr->at(0) != static_cast(log_surgeon::SymbolId::TokenEnd)) { - SPDLOG_INFO("token:" + token.to_string() + "\n"); - SPDLOG_INFO( - "token.m_type_ids->back():" - + reverse_lexer.m_id_symbol[token.m_type_ids_ptr->back()] + "\n" - ); - auto [error_code, opt_token] = reverse_lexer.scan(parser_input_buffer); + auto [error_code, opt_token] = lexer.scan(parser_input_buffer); REQUIRE(error_code == log_surgeon::ErrorCode::Success); token = opt_token.value(); } From 5cc0b147ad942fc58c5d2ec6252666c0e3e0969f Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Wed, 16 Jul 2025 13:03:59 -0400 Subject: [PATCH 28/49] Lint. --- components/core/src/clp/Grep.cpp | 6 +- components/core/src/clp/clg/clg.cpp | 10 +- components/core/tests/test-Grep.cpp | 148 ++---------------- .../core/tests/test-ParserWithUserSchema.cpp | 3 +- 4 files changed, 21 insertions(+), 146 deletions(-) diff --git a/components/core/src/clp/Grep.cpp b/components/core/src/clp/Grep.cpp index ae03d9dfb3..d1503d6e2d 100644 --- a/components/core/src/clp/Grep.cpp +++ b/components/core/src/clp/Grep.cpp @@ -842,11 +842,7 @@ bool Grep::get_bounds_of_next_potential_var( string_reader.open(value.substr(begin_pos, end_pos - begin_pos - 1)); parser_input_buffer.read_if_safe(reader_wrapper); lexer.reset(); - lexer.scan_with_wildcard( - parser_input_buffer, - value[end_pos - 1], - search_token - ); + lexer.scan_with_wildcard(parser_input_buffer, value[end_pos - 1], search_token); } else { // no wildcards string_reader.open(value.substr(begin_pos, end_pos - begin_pos)); parser_input_buffer.read_if_safe(reader_wrapper); diff --git a/components/core/src/clp/clg/clg.cpp b/components/core/src/clp/clg/clg.cpp index 3b21ce902c..8144707fba 100644 --- a/components/core/src/clp/clg/clg.cpp +++ b/components/core/src/clp/clg/clg.cpp @@ -591,8 +591,7 @@ int main(int argc, char const* argv[]) { // if there is a chance there might be a difference make a new lexer as it's pretty // fast to create if (lexer_map_it == lexer_map.end()) { - auto insert_result - = lexer_map.emplace(buf, log_surgeon::lexers::ByteLexer()); + auto insert_result = lexer_map.emplace(buf, log_surgeon::lexers::ByteLexer()); lexer_ptr = &insert_result.first->second; load_lexer_from_file(schema_file_path, *lexer_ptr); } else { @@ -605,12 +604,7 @@ int main(int argc, char const* argv[]) { } // Perform search - if (!search(search_strings, - command_line_args, - archive_reader, - *lexer_ptr, - use_heuristic)) - { + if (!search(search_strings, command_line_args, archive_reader, *lexer_ptr, use_heuristic)) { return -1; } archive_reader.close(); diff --git a/components/core/tests/test-Grep.cpp b/components/core/tests/test-Grep.cpp index 69eb27aee6..e4a256c5d5 100644 --- a/components/core/tests/test-Grep.cpp +++ b/components/core/tests/test-Grep.cpp @@ -29,39 +29,21 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var str = ""; begin_pos = string::npos; end_pos = string::npos; - REQUIRE(Grep::get_bounds_of_next_potential_var( - str, - begin_pos, - end_pos, - is_var, - lexer - ) + REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == false); // Empty string str = ""; begin_pos = 0; end_pos = 0; - REQUIRE(Grep::get_bounds_of_next_potential_var( - str, - begin_pos, - end_pos, - is_var, - lexer - ) + REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == false); // No tokens str = "="; begin_pos = 0; end_pos = 0; - REQUIRE(Grep::get_bounds_of_next_potential_var( - str, - begin_pos, - end_pos, - is_var, - lexer - ) + REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == false); // No wildcards @@ -69,80 +51,31 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos = 0; end_pos = 0; - REQUIRE(Grep::get_bounds_of_next_potential_var( - str, - begin_pos, - end_pos, - is_var, - lexer - ) - == true); + REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); REQUIRE("95" == str.substr(begin_pos, end_pos - begin_pos)); REQUIRE(true == is_var); - REQUIRE(Grep::get_bounds_of_next_potential_var( - str, - begin_pos, - end_pos, - is_var, - lexer - ) - == true); + REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); REQUIRE("ad" == str.substr(begin_pos, end_pos - begin_pos)); REQUIRE(true == is_var); - REQUIRE(Grep::get_bounds_of_next_potential_var( - str, - begin_pos, - end_pos, - is_var, - lexer - ) - == true); + REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); REQUIRE("ff" == str.substr(begin_pos, end_pos - begin_pos)); REQUIRE(true == is_var); - REQUIRE(Grep::get_bounds_of_next_potential_var( - str, - begin_pos, - end_pos, - is_var, - lexer - ) - == true); + REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); REQUIRE("95" == str.substr(begin_pos, end_pos - begin_pos)); REQUIRE(true == is_var); - REQUIRE(Grep::get_bounds_of_next_potential_var( - str, - begin_pos, - end_pos, - is_var, - lexer - ) - == true); + REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); REQUIRE("24" == str.substr(begin_pos, end_pos - begin_pos)); REQUIRE(true == is_var); - REQUIRE(Grep::get_bounds_of_next_potential_var( - str, - begin_pos, - end_pos, - is_var, - lexer - ) - == true); + REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); REQUIRE("0d" == str.substr(begin_pos, end_pos - begin_pos)); REQUIRE(true == is_var); - REQUIRE(Grep::get_bounds_of_next_potential_var( - str, - begin_pos, - end_pos, - is_var, - lexer - ) - == true); + REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); REQUIRE("ff" == str.substr(begin_pos, end_pos - begin_pos)); REQUIRE(true == is_var); @@ -150,13 +83,7 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var REQUIRE("-abc-" == str.substr(begin_pos, end_pos - begin_pos)); REQUIRE(true == is_var); - REQUIRE(Grep::get_bounds_of_next_potential_var( - str, - begin_pos, - end_pos, - is_var, - lexer - ) + REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == false); REQUIRE(str.length() == begin_pos); @@ -165,69 +92,28 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos = 0; end_pos = 0; - REQUIRE(Grep::get_bounds_of_next_potential_var( - str, - begin_pos, - end_pos, - is_var, - lexer - ) - == true); + REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); REQUIRE(str.substr(begin_pos, end_pos - begin_pos) == "1\\*x"); REQUIRE(is_var == true); // REQUIRE(is_var == true); - REQUIRE(Grep::get_bounds_of_next_potential_var( - str, - begin_pos, - end_pos, - is_var, - lexer - ) - == true); + REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); REQUIRE(str.substr(begin_pos, end_pos - begin_pos) == "abc*123"); REQUIRE(is_var == false); // REQUIRE(is_var == true); - REQUIRE(Grep::get_bounds_of_next_potential_var( - str, - begin_pos, - end_pos, - is_var, - lexer - ) - == true); + REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); REQUIRE(str.substr(begin_pos, end_pos - begin_pos) == "1.2"); REQUIRE(is_var == true); - REQUIRE(Grep::get_bounds_of_next_potential_var( - str, - begin_pos, - end_pos, - is_var, - lexer - ) - == true); + REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); REQUIRE(str.substr(begin_pos, end_pos - begin_pos) == "+394/-"); REQUIRE(is_var == true); - REQUIRE(Grep::get_bounds_of_next_potential_var( - str, - begin_pos, - end_pos, - is_var, - lexer - ) - == true); + REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); REQUIRE(str.substr(begin_pos, end_pos - begin_pos) == "-*abc-"); REQUIRE(is_var == false); - REQUIRE(Grep::get_bounds_of_next_potential_var( - str, - begin_pos, - end_pos, - is_var, - lexer - ) + REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == false); } diff --git a/components/core/tests/test-ParserWithUserSchema.cpp b/components/core/tests/test-ParserWithUserSchema.cpp index 844c7835a9..7b90e90e64 100644 --- a/components/core/tests/test-ParserWithUserSchema.cpp +++ b/components/core/tests/test-ParserWithUserSchema.cpp @@ -171,8 +171,7 @@ TEST_CASE("Test lexer", "[Search]") { while (token.m_type_ids_ptr->at(0) != static_cast(log_surgeon::SymbolId::TokenEnd)) { SPDLOG_INFO("token:" + token.to_string() + "\n"); SPDLOG_INFO( - "token.m_type_ids->back():" - + lexer.m_id_symbol[token.m_type_ids_ptr->back()] + "\n" + "token.m_type_ids->back():" + lexer.m_id_symbol[token.m_type_ids_ptr->back()] + "\n" ); auto [error_code, opt_token] = lexer.scan(parser_input_buffer); REQUIRE(error_code == log_surgeon::ErrorCode::Success); From 23de2c347e9a7a9013b63263ff58cad5b2ce1cae Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Wed, 16 Jul 2025 13:04:24 -0400 Subject: [PATCH 29/49] Remove TODO. --- components/core/src/clp/Grep.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/components/core/src/clp/Grep.cpp b/components/core/src/clp/Grep.cpp index d1503d6e2d..e0c2caf51c 100644 --- a/components/core/src/clp/Grep.cpp +++ b/components/core/src/clp/Grep.cpp @@ -854,15 +854,6 @@ bool Grep::get_bounds_of_next_potential_var( search_token = SearchToken{token.value()}; search_token.m_type_ids_set.insert(search_token.m_type_ids_ptr->at(0)); } - // TODO: use a set so its faster - // auto const& set = search_token.m_type_ids_set; - // if (set.find(static_cast(log_surgeon::SymbolId::TokenUncaughtStringID)) - // == set.end() - // && set.find(static_cast(log_surgeon::SymbolId::TokenEndID)) - // == set.end()) - // { - // is_var = true; - // } auto const& type = search_token.m_type_ids_ptr->at(0); if (type != static_cast(log_surgeon::SymbolId::TokenUncaughtString) && type != static_cast(log_surgeon::SymbolId::TokenEnd)) From bd2ae8e45341c1a6cb14cf02edfa9b4db60c259a Mon Sep 17 00:00:00 2001 From: davidlion Date: Wed, 16 Jul 2025 22:17:12 -0400 Subject: [PATCH 30/49] Add heading comments to schemas. --- components/core/config/schemas.txt | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index 1d3a36c3f1..f254c4a4b5 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -1,4 +1,6 @@ +// === // Timestamps +// === // E.g. 2015-01-31T15:50:45.392 timestamp:\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}.\d{3} @@ -71,14 +73,25 @@ timestamp:\d{2}\-\d{2} \d{2}:\d{2}:\d{2}.\d{3} // E.g. 2016-05-08 11:34:04.083464 timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}.\d{6} +// === // Delimiters +// === + delimiters: \t\r\n!"#$%&'\(\)\*,:;\<\>\?@\[\]\^_`\{\|\}~= +// === // Specially-encoded variables (using the `int` and `float` keywords) +// === + int:\-{0,1}[0-9]+ float:\-{0,1}[0-9]+\.[0-9]+ -// Dictionary variables (the way `equals` and `hasNumber` are written only work if `=` is a delim) +// === +// Dictionary variables +// === + hex:[a-fA-F]+ + +// Note: the way `equals` and `hasNumber` are written only work if `=` is a delim equals:.*=(?.*[a-zA-Z0-9].*) hasNumber:.*\d.* From d43c4c7afd3f30b4d0c8bc46ff1b4e0d1a70bbe5 Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Thu, 17 Jul 2025 11:57:55 -0400 Subject: [PATCH 31/49] Update schema timestamps to escape '.'. --- components/core/config/schemas.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index f254c4a4b5..9a706f2697 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -3,7 +3,7 @@ // === // E.g. 2015-01-31T15:50:45.392 -timestamp:\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}.\d{3} +timestamp:\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3} // E.g. 2015-01-31T15:50:45,392 timestamp:\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2},\d{3} @@ -19,7 +19,7 @@ timestamp:\[\d{4}\d{2}\d{2}\-\d{2}:\d{2}:\d{2}\] timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2},\d{3} // E.g. 2015-01-31 15:50:45.392 -timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}.\d{3} +timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} // E.g. [2015-01-31 15:50:45,085] timestamp:\[\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2},\d{3}\] @@ -68,10 +68,10 @@ timestamp:\<\<\<\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}:\d{3} timestamp:[A-Z][a-z]{2} \d{2} \d{2}:\d{2}:\d{2} // E.g. 01-21 11:56:42.392 -timestamp:\d{2}\-\d{2} \d{2}:\d{2}:\d{2}.\d{3} +timestamp:\d{2}\-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} // E.g. 2016-05-08 11:34:04.083464 -timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}.\d{6} +timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}\.\d{6} // === // Delimiters From d5574672272f38ef1d507439dce0df3ff60b5436 Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Thu, 17 Jul 2025 12:31:23 -0400 Subject: [PATCH 32/49] Update regex in schema for leading spaces. --- components/core/config/schemas.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index 9a706f2697..5199946b5f 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -39,13 +39,13 @@ timestamp:\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} timestamp:\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} // E.g. 150131 9:50:45 -timestamp:\d{2}\d{2}\d{2} [ 0-9]{2}:\d{2}:\d{2} +timestamp:\d{2}\d{2}\d{2} [ \d]\d:\d{2}:\d{2} // E.g. 01 Jan 2016 15:50:17,085 timestamp:\d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2},\d{3} // E.g. Jan 01, 2016 3:50:17 PM -timestamp:[A-Z][a-z]{2} \d{2}, \d{4} [ 0-9]{2}:\d{2}:\d{2} [AP]M +timestamp:[A-Z][a-z]{2} \d{2}, \d{4} [ \d]\d:\d{2}:\d{2} [AP]M // E.g. January 31, 2015 15:50 timestamp:[A-Z][a-z]+ \d{2}, \d{4} \d{2}:\d{2} @@ -59,7 +59,7 @@ timestamp:\[\d{2}/[A-Z][a-z]{2}/\d{4}:\d{2}:\d{2}:\d{2} timestamp:\[\d{2}/\d{2}/\d{4}:\d{2}:\d{2}:\d{2} // E.g. ERROR: apport (pid 4557) Sun Jan 1 15:50:45 2015 -timestamp:[A-Z][a-z]{2} [A-Z][a-z]{2} [ 0-9]{2} \d{2}:\d{2}:\d{2} \d{4} +timestamp:[A-Z][a-z]{2} [A-Z][a-z]{2} [ \d]\d \d{2}:\d{2}:\d{2} \d{4} // E.g. <<<2016-11-10 03:02:29:936 timestamp:\<\<\<\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}:\d{3} From a41ee92eb08ba2c5e175bfcb6434d7959935b019 Mon Sep 17 00:00:00 2001 From: davidlion Date: Fri, 18 Jul 2025 21:17:58 -0400 Subject: [PATCH 33/49] Add issue link in schemas.txt. --- components/core/config/schemas.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index f254c4a4b5..c35007caf9 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -92,6 +92,7 @@ float:\-{0,1}[0-9]+\.[0-9]+ hex:[a-fA-F]+ -// Note: the way `equals` and `hasNumber` are written only work if `=` is a delim +// Note: the way `equals` and `hasNumber` are written only work if `=` is a delimiter, due to +// https://github.com/y-scope/log-surgeon/issues/131 equals:.*=(?.*[a-zA-Z0-9].*) hasNumber:.*\d.* From fbf619511e5853d5e17510a0eac503be22e6eb0d Mon Sep 17 00:00:00 2001 From: davidlion Date: Mon, 21 Jul 2025 00:50:53 -0400 Subject: [PATCH 34/49] Make lexer safer in clo; Remove dead declaration in clg. --- components/core/src/clp/clg/clg.cpp | 13 ------------- components/core/src/clp/clo/clo.cpp | 4 ++-- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/components/core/src/clp/clg/clg.cpp b/components/core/src/clp/clg/clg.cpp index 8144707fba..5faa8e0cc8 100644 --- a/components/core/src/clp/clg/clg.cpp +++ b/components/core/src/clp/clg/clg.cpp @@ -48,19 +48,6 @@ using std::vector; * @return true on success, false otherwise */ static bool open_archive(string const& archive_path, Archive& archive_reader); -/** - * Searches the archive with the given parameters - * @param search_strings - * @param command_line_args - * @param archive - * @return true on success, false otherwise - */ -static bool search( - vector const& search_strings, - CommandLineArguments& command_line_args, - Archive& archive, - bool use_heuristic -); /** * Opens a compressed file or logs any errors if it couldn't be opened * @param file_metadata_ix diff --git a/components/core/src/clp/clo/clo.cpp b/components/core/src/clp/clo/clo.cpp index a8118cdbb2..6a8dfe59d9 100644 --- a/components/core/src/clp/clo/clo.cpp +++ b/components/core/src/clp/clo/clo.cpp @@ -476,11 +476,11 @@ static bool search_archive( // Load lexers from schema file if it exists auto schema_file_path = archive_path / clp::streaming_archive::cSchemaFileName; - unique_ptr lexer; + auto lexer{std::make_unique()}; bool use_heuristic = true; if (std::filesystem::exists(schema_file_path)) { use_heuristic = false; - lexer.reset(new log_surgeon::lexers::ByteLexer()); + lexer = std::make_unique(); load_lexer_from_file(schema_file_path.string(), *lexer); } From d1756ef108046345714e4b8c7fe33bfeb47ec249 Mon Sep 17 00:00:00 2001 From: davidlion Date: Mon, 21 Jul 2025 00:55:49 -0400 Subject: [PATCH 35/49] Update components/core/config/schemas.txt Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/config/schemas.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index 5c966a3848..5f9b69c67f 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -77,7 +77,7 @@ timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}\.\d{6} // Delimiters // === -delimiters: \t\r\n!"#$%&'\(\)\*,:;\<\>\?@\[\]\^_`\{\|\}~= +delimiters: \t\r\n!"#$%&'\(\)\*,:;\<=\>\?@\[\]\^_`\{\|\}~ // === // Specially-encoded variables (using the `int` and `float` keywords) From 8d12a2bd7f9d2c4a8d7a8041720b3166544b55f7 Mon Sep 17 00:00:00 2001 From: davidlion Date: Mon, 21 Jul 2025 00:56:24 -0400 Subject: [PATCH 36/49] Update components/core/config/schemas.txt Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/config/schemas.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index 5f9b69c67f..c522bdd3ed 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -92,7 +92,7 @@ float:\-{0,1}[0-9]+\.[0-9]+ hex:[a-fA-F]+ -// Note: the way `equals` and `hasNumber` are written only work if `=` is a delimiter, due to +// NOTE: `equals` and `hasNumber` only work if `=` is a delimiter, due to // https://github.com/y-scope/log-surgeon/issues/131 equals:.*=(?.*[a-zA-Z0-9].*) hasNumber:.*\d.* From 46bc35ffb700f42049b574308d91a6ccfa538522 Mon Sep 17 00:00:00 2001 From: davidlion Date: Mon, 21 Jul 2025 00:59:09 -0400 Subject: [PATCH 37/49] Update components/core/config/schemas.txt Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/config/schemas.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index c522bdd3ed..4fe0aa4d4b 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -3,7 +3,10 @@ // === // E.g. 2015-01-31T15:50:45.392 -timestamp:\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3} +// E.g. 2015-01-31T15:50:45,392 +// E.g. 2015-01-31 15:50:45.392 +// E.g. 2015-01-31 15:50:45,392 +timestamp:\d{4}\-\d{2}\-\d{2}[T ]\d{2}:\d{2}:\d{2}[,\.]\d{3} // E.g. 2015-01-31T15:50:45,392 timestamp:\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2},\d{3} From 063297d2c4a898cbe539454feb20b1d007143a6c Mon Sep 17 00:00:00 2001 From: davidlion Date: Mon, 21 Jul 2025 01:01:18 -0400 Subject: [PATCH 38/49] Delete redundant timestamps. --- components/core/config/schemas.txt | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index 4fe0aa4d4b..f9fd5368e0 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -8,22 +8,12 @@ // E.g. 2015-01-31 15:50:45,392 timestamp:\d{4}\-\d{2}\-\d{2}[T ]\d{2}:\d{2}:\d{2}[,\.]\d{3} -// E.g. 2015-01-31T15:50:45,392 -timestamp:\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2},\d{3} - // E.g. [2015-01-31T15:50:45 timestamp:\[\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2} // E.g. [20170106-16:56:41] timestamp:\[\d{4}\d{2}\d{2}\-\d{2}:\d{2}:\d{2}\] -// E.g. 2015-01-31 15:50:45,392 -// E.g. INFO [main] 2015-01-31 15:50:45,085 -timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2},\d{3} - -// E.g. 2015-01-31 15:50:45.392 -timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} - // E.g. [2015-01-31 15:50:45,085] timestamp:\[\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2},\d{3}\] From 75dab674d935b3c29add369bbab91ef2994c129e Mon Sep 17 00:00:00 2001 From: davidlion Date: Mon, 21 Jul 2025 01:05:03 -0400 Subject: [PATCH 39/49] Allow dates with space + single digit. --- components/core/config/schemas.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index f9fd5368e0..8ee99a5ede 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -35,10 +35,10 @@ timestamp:\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} timestamp:\d{2}\d{2}\d{2} [ \d]\d:\d{2}:\d{2} // E.g. 01 Jan 2016 15:50:17,085 -timestamp:\d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2},\d{3} +timestamp:[ \d]\d [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2},\d{3} // E.g. Jan 01, 2016 3:50:17 PM -timestamp:[A-Z][a-z]{2} \d{2}, \d{4} [ \d]\d:\d{2}:\d{2} [AP]M +timestamp:[A-Z][a-z]{2} [ \d]\d, \d{4} [ \d]\d:\d{2}:\d{2} [AP]M // E.g. January 31, 2015 15:50 timestamp:[A-Z][a-z]+ \d{2}, \d{4} \d{2}:\d{2} @@ -58,7 +58,7 @@ timestamp:[A-Z][a-z]{2} [A-Z][a-z]{2} [ \d]\d \d{2}:\d{2}:\d{2} \d{4} timestamp:\<\<\<\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}:\d{3} // E.g. Jan 21 11:56:42 -timestamp:[A-Z][a-z]{2} \d{2} \d{2}:\d{2}:\d{2} +timestamp:[A-Z][a-z]{2} [ \d]\d \d{2}:\d{2}:\d{2} // E.g. 01-21 11:56:42.392 timestamp:\d{2}\-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} From 716e8715beb11d8a95e796de67ed134884530e2c Mon Sep 17 00:00:00 2001 From: davidlion Date: Mon, 21 Jul 2025 01:56:22 -0400 Subject: [PATCH 40/49] Tweak schemas. --- components/core/config/schemas.txt | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index 8ee99a5ede..8954ffad2f 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -6,7 +6,21 @@ // E.g. 2015-01-31T15:50:45,392 // E.g. 2015-01-31 15:50:45.392 // E.g. 2015-01-31 15:50:45,392 -timestamp:\d{4}\-\d{2}\-\d{2}[T ]\d{2}:\d{2}:\d{2}[,\.]\d{3} +// E.g. 2015/01/31T15:50:45.123 +// E.g. 2015/01/31T15:50:45,123 +// E.g. 2015/01/31 15:50:45.123 +// E.g. 2015/01/31 15:50:45,123 +timestamp:\d{4}[\-/]\d{2}[\-/]\d{2}[T ]\d{2}:\d{2}:\d{2}[,\.]\d{3} + +// E.g. 2015-01-31T15:50:45 +// E.g. 2015-01-31T15:50:45 +// E.g. 2015-01-31 15:50:45 +// E.g. 2015-01-31 15:50:45 +// E.g. 2015/01/31T15:50:45 +// E.g. 2015/01/31T15:50:45 +// E.g. 2015/01/31 15:50:45 +// E.g. 2015/01/31 15:50:45 +timestamp:\d{4}[\-/]\d{2}[\-/]\d{2}[T ]\d{2}:\d{2}:\d{2} // E.g. [2015-01-31T15:50:45 timestamp:\[\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2} @@ -25,9 +39,6 @@ timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2} // E.g. Start-Date: 2015-01-31 15:50:45 timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2} -// E.g. 2015/01/31 15:50:45 -timestamp:\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} - // E.g. 15/01/31 15:50:45 timestamp:\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} From 70db06f30948bd2867276ce2d4355ba3a4aff002 Mon Sep 17 00:00:00 2001 From: davidlion Date: Mon, 21 Jul 2025 11:11:29 -0400 Subject: [PATCH 41/49] Consolidate timestamps into fewer regex supporting many more combinations/options. --- components/core/config/schemas.txt | 55 +++++++----------------------- 1 file changed, 13 insertions(+), 42 deletions(-) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index 8954ffad2f..9e6a77fb79 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -2,36 +2,19 @@ // Timestamps // === -// E.g. 2015-01-31T15:50:45.392 -// E.g. 2015-01-31T15:50:45,392 -// E.g. 2015-01-31 15:50:45.392 -// E.g. 2015-01-31 15:50:45,392 -// E.g. 2015/01/31T15:50:45.123 -// E.g. 2015/01/31T15:50:45,123 -// E.g. 2015/01/31 15:50:45.123 -// E.g. 2015/01/31 15:50:45,123 -timestamp:\d{4}[\-/]\d{2}[\-/]\d{2}[T ]\d{2}:\d{2}:\d{2}[,\.]\d{3} - -// E.g. 2015-01-31T15:50:45 -// E.g. 2015-01-31T15:50:45 -// E.g. 2015-01-31 15:50:45 -// E.g. 2015-01-31 15:50:45 -// E.g. 2015/01/31T15:50:45 -// E.g. 2015/01/31T15:50:45 -// E.g. 2015/01/31 15:50:45 -// E.g. 2015/01/31 15:50:45 -timestamp:\d{4}[\-/]\d{2}[\-/]\d{2}[T ]\d{2}:\d{2}:\d{2} - -// E.g. [2015-01-31T15:50:45 -timestamp:\[\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2} - -// E.g. [20170106-16:56:41] -timestamp:\[\d{4}\d{2}\d{2}\-\d{2}:\d{2}:\d{2}\] - -// E.g. [2015-01-31 15:50:45,085] -timestamp:\[\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2},\d{3}\] - -// E.g. 2015-01-31 15:50:45 +// E.g. 2015-01-31T15:50:45,123 +// supported options: +// - date punctuation: "2015-01-31" "2015/01/31" "20150131" +// - date-time separator: "31 15" "31-15" "31T15" +// - enclosing symbols: +// - prefixes: "2015" "<<<2015" "[2015" +// - suffixes: ":45,123" ":45,123]" +// - hour leading digit: "T05:50" "T 5:50" +// - precision from tenths to microseconds: ":45,123" ":45,123456" +// - separator: ":45,123" ":45.123" ":45:123" +// - year notation: "2015-01" "15-01" +timestamp:(\<{3}|\[){0,1}(\d{2}|\d{4})[\-/]{0,1}\d{2}[\-/]{0,1}\d{2}[ -T][ \d]\d:\d{2}:\d{2}([,\.:]\d{1,6}){0,1}(\]){0,1} + // E.g. Started POST /api/v3/internal/allowed for 127.0.0.1 at 2017-06-18 00:20:44 // E.g. update-alternatives 2015-01-31 15:50:45 timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2} @@ -39,12 +22,6 @@ timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2} // E.g. Start-Date: 2015-01-31 15:50:45 timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2} -// E.g. 15/01/31 15:50:45 -timestamp:\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} - -// E.g. 150131 9:50:45 -timestamp:\d{2}\d{2}\d{2} [ \d]\d:\d{2}:\d{2} - // E.g. 01 Jan 2016 15:50:17,085 timestamp:[ \d]\d [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2},\d{3} @@ -65,18 +42,12 @@ timestamp:\[\d{2}/\d{2}/\d{4}:\d{2}:\d{2}:\d{2} // E.g. ERROR: apport (pid 4557) Sun Jan 1 15:50:45 2015 timestamp:[A-Z][a-z]{2} [A-Z][a-z]{2} [ \d]\d \d{2}:\d{2}:\d{2} \d{4} -// E.g. <<<2016-11-10 03:02:29:936 -timestamp:\<\<\<\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}:\d{3} - // E.g. Jan 21 11:56:42 timestamp:[A-Z][a-z]{2} [ \d]\d \d{2}:\d{2}:\d{2} // E.g. 01-21 11:56:42.392 timestamp:\d{2}\-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} -// E.g. 2016-05-08 11:34:04.083464 -timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}\.\d{6} - // === // Delimiters // === From 64c0216ec451b0dd3329ae50badda5fec3a17ca8 Mon Sep 17 00:00:00 2001 From: davidlion Date: Mon, 21 Jul 2025 11:20:19 -0400 Subject: [PATCH 42/49] Add missing date case caught by rabbit. --- components/core/config/schemas.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index 9e6a77fb79..2626ab8499 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -29,7 +29,7 @@ timestamp:[ \d]\d [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2},\d{3} timestamp:[A-Z][a-z]{2} [ \d]\d, \d{4} [ \d]\d:\d{2}:\d{2} [AP]M // E.g. January 31, 2015 15:50 -timestamp:[A-Z][a-z]+ \d{2}, \d{4} \d{2}:\d{2} +timestamp:[A-Z][a-z]+ [ \d]\d, \d{4} \d{2}:\d{2} // E.g. E [31/Jan/2015:15:50:45 // E.g. localhost - - [01/Jan/2016:15:50:17 From 4a05836c6c1b1684c2ddd12cce40abe404efb921 Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Mon, 21 Jul 2025 12:03:51 -0400 Subject: [PATCH 43/49] Avoid recreating the lexer object. --- components/core/src/clp/clo/clo.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/core/src/clp/clo/clo.cpp b/components/core/src/clp/clo/clo.cpp index 6a8dfe59d9..0f75b92df3 100644 --- a/components/core/src/clp/clo/clo.cpp +++ b/components/core/src/clp/clo/clo.cpp @@ -480,7 +480,6 @@ static bool search_archive( bool use_heuristic = true; if (std::filesystem::exists(schema_file_path)) { use_heuristic = false; - lexer = std::make_unique(); load_lexer_from_file(schema_file_path.string(), *lexer); } From 4a730a4092a4678af626331d1ca5ff21484f3e1e Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Mon, 21 Jul 2025 12:34:41 -0400 Subject: [PATCH 44/49] Use Lexer object directly instead of a unique_ptr. --- components/core/src/clp/clo/clo.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/core/src/clp/clo/clo.cpp b/components/core/src/clp/clo/clo.cpp index 0f75b92df3..b94ffb9555 100644 --- a/components/core/src/clp/clo/clo.cpp +++ b/components/core/src/clp/clo/clo.cpp @@ -476,11 +476,11 @@ static bool search_archive( // Load lexers from schema file if it exists auto schema_file_path = archive_path / clp::streaming_archive::cSchemaFileName; - auto lexer{std::make_unique()}; + log_surgeon::lexers::ByteLexer lexer; bool use_heuristic = true; if (std::filesystem::exists(schema_file_path)) { use_heuristic = false; - load_lexer_from_file(schema_file_path.string(), *lexer); + load_lexer_from_file(schema_file_path.string(), lexer); } Archive archive_reader; @@ -496,7 +496,7 @@ static bool search_archive( search_begin_ts, search_end_ts, command_line_args.ignore_case(), - *lexer, + lexer, use_heuristic ); if (false == query_processing_result.has_value()) { From d654f4623fc252a661573a07c5a198641a543b02 Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Mon, 21 Jul 2025 13:30:45 -0400 Subject: [PATCH 45/49] Throw error if parsed timestamp can't be encoded. --- components/core/src/clp/streaming_archive/writer/Archive.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/core/src/clp/streaming_archive/writer/Archive.cpp b/components/core/src/clp/streaming_archive/writer/Archive.cpp index 1a6010b971..79f07a0358 100644 --- a/components/core/src/clp/streaming_archive/writer/Archive.cpp +++ b/components/core/src/clp/streaming_archive/writer/Archive.cpp @@ -328,6 +328,9 @@ void Archive::write_msg_using_schema(LogEventView const& log_view) { start, end ); + if(timestamp_pattern == nullptr) { + throw(std::runtime_error("Timestamp encoding failed")); + } if (m_old_ts_pattern != timestamp_pattern) { change_ts_pattern(timestamp_pattern); m_old_ts_pattern = timestamp_pattern; From a02efeee663b7c546481b9b61ff535268a82a437 Mon Sep 17 00:00:00 2001 From: davidlion Date: Mon, 21 Jul 2025 13:35:15 -0400 Subject: [PATCH 46/49] Reogranize schema slightly. --- components/core/config/schemas.txt | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index 2626ab8499..d2e6d5e7d8 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -15,13 +15,6 @@ // - year notation: "2015-01" "15-01" timestamp:(\<{3}|\[){0,1}(\d{2}|\d{4})[\-/]{0,1}\d{2}[\-/]{0,1}\d{2}[ -T][ \d]\d:\d{2}:\d{2}([,\.:]\d{1,6}){0,1}(\]){0,1} -// E.g. Started POST /api/v3/internal/allowed for 127.0.0.1 at 2017-06-18 00:20:44 -// E.g. update-alternatives 2015-01-31 15:50:45 -timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2} - -// E.g. Start-Date: 2015-01-31 15:50:45 -timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2} - // E.g. 01 Jan 2016 15:50:17,085 timestamp:[ \d]\d [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2},\d{3} @@ -31,6 +24,23 @@ timestamp:[A-Z][a-z]{2} [ \d]\d, \d{4} [ \d]\d:\d{2}:\d{2} [AP]M // E.g. January 31, 2015 15:50 timestamp:[A-Z][a-z]+ [ \d]\d, \d{4} \d{2}:\d{2} +// E.g. Jan 21 11:56:42 +timestamp:[A-Z][a-z]{2} [ \d]\d \d{2}:\d{2}:\d{2} + +// E.g. 01-21 11:56:42.392 +timestamp:\d{2}\-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} + +// === +// Timestamps currently missing log boundary support +// === + +// E.g. Started POST /api/v3/internal/allowed for 127.0.0.1 at 2017-06-18 00:20:44 +// E.g. update-alternatives 2015-01-31 15:50:45 +timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2} + +// E.g. Start-Date: 2015-01-31 15:50:45 +timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2} + // E.g. E [31/Jan/2015:15:50:45 // E.g. localhost - - [01/Jan/2016:15:50:17 // E.g. 192.168.4.5 - - [01/Jan/2016:15:50:17 @@ -42,12 +52,6 @@ timestamp:\[\d{2}/\d{2}/\d{4}:\d{2}:\d{2}:\d{2} // E.g. ERROR: apport (pid 4557) Sun Jan 1 15:50:45 2015 timestamp:[A-Z][a-z]{2} [A-Z][a-z]{2} [ \d]\d \d{2}:\d{2}:\d{2} \d{4} -// E.g. Jan 21 11:56:42 -timestamp:[A-Z][a-z]{2} [ \d]\d \d{2}:\d{2}:\d{2} - -// E.g. 01-21 11:56:42.392 -timestamp:\d{2}\-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} - // === // Delimiters // === From 82f982561a7ec43d9ec6fa1c329f9fcf76c54c9a Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Mon, 21 Jul 2025 13:38:03 -0400 Subject: [PATCH 47/49] Update error message; Reorded if statement; Lint. --- .../core/src/clp/streaming_archive/writer/Archive.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp/streaming_archive/writer/Archive.cpp b/components/core/src/clp/streaming_archive/writer/Archive.cpp index 79f07a0358..a5fe8a29f9 100644 --- a/components/core/src/clp/streaming_archive/writer/Archive.cpp +++ b/components/core/src/clp/streaming_archive/writer/Archive.cpp @@ -328,8 +328,12 @@ void Archive::write_msg_using_schema(LogEventView const& log_view) { start, end ); - if(timestamp_pattern == nullptr) { - throw(std::runtime_error("Timestamp encoding failed")); + if (nullptr == timestamp_pattern) { + throw(std::runtime_error( + "Schema contains a timestamp regex that matches " + + log_output_buffer->get_mutable_token(0).to_string() + + " which does not match any known timestamp pattern." + )); } if (m_old_ts_pattern != timestamp_pattern) { change_ts_pattern(timestamp_pattern); From 922053be5c86349c7e3662b471c4d18f37da0c1b Mon Sep 17 00:00:00 2001 From: SharafMohamed Date: Mon, 21 Jul 2025 14:08:58 -0400 Subject: [PATCH 48/49] Add relative timestamp regex. --- components/core/config/schemas.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index d2e6d5e7d8..ef2da119f2 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -30,6 +30,9 @@ timestamp:[A-Z][a-z]{2} [ \d]\d \d{2}:\d{2}:\d{2} // E.g. 01-21 11:56:42.392 timestamp:\d{2}\-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} +// E.g. 916321 +timestamp:\d+ + // === // Timestamps currently missing log boundary support // === From 09889d6f83a3fb3ea562e571b130935849924f26 Mon Sep 17 00:00:00 2001 From: davidlion Date: Fri, 25 Jul 2025 10:16:55 -0400 Subject: [PATCH 49/49] Revert schemas.txt. --- components/core/config/schemas.txt | 76 ++++-------------------------- 1 file changed, 9 insertions(+), 67 deletions(-) diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index d2e6d5e7d8..e0b777859d 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -1,77 +1,19 @@ -// === -// Timestamps -// === - -// E.g. 2015-01-31T15:50:45,123 -// supported options: -// - date punctuation: "2015-01-31" "2015/01/31" "20150131" -// - date-time separator: "31 15" "31-15" "31T15" -// - enclosing symbols: -// - prefixes: "2015" "<<<2015" "[2015" -// - suffixes: ":45,123" ":45,123]" -// - hour leading digit: "T05:50" "T 5:50" -// - precision from tenths to microseconds: ":45,123" ":45,123456" -// - separator: ":45,123" ":45.123" ":45:123" -// - year notation: "2015-01" "15-01" -timestamp:(\<{3}|\[){0,1}(\d{2}|\d{4})[\-/]{0,1}\d{2}[\-/]{0,1}\d{2}[ -T][ \d]\d:\d{2}:\d{2}([,\.:]\d{1,6}){0,1}(\]){0,1} - -// E.g. 01 Jan 2016 15:50:17,085 -timestamp:[ \d]\d [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2},\d{3} - -// E.g. Jan 01, 2016 3:50:17 PM -timestamp:[A-Z][a-z]{2} [ \d]\d, \d{4} [ \d]\d:\d{2}:\d{2} [AP]M - -// E.g. January 31, 2015 15:50 -timestamp:[A-Z][a-z]+ [ \d]\d, \d{4} \d{2}:\d{2} - -// E.g. Jan 21 11:56:42 -timestamp:[A-Z][a-z]{2} [ \d]\d \d{2}:\d{2}:\d{2} - -// E.g. 01-21 11:56:42.392 -timestamp:\d{2}\-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} - -// === -// Timestamps currently missing log boundary support -// === - -// E.g. Started POST /api/v3/internal/allowed for 127.0.0.1 at 2017-06-18 00:20:44 -// E.g. update-alternatives 2015-01-31 15:50:45 -timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2} - -// E.g. Start-Date: 2015-01-31 15:50:45 -timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2} - -// E.g. E [31/Jan/2015:15:50:45 -// E.g. localhost - - [01/Jan/2016:15:50:17 -// E.g. 192.168.4.5 - - [01/Jan/2016:15:50:17 -timestamp:\[\d{2}/[A-Z][a-z]{2}/\d{4}:\d{2}:\d{2}:\d{2} - -// E.g. 192.168.4.5 - - [01/01/2016:15:50:17 -timestamp:\[\d{2}/\d{2}/\d{4}:\d{2}:\d{2}:\d{2} - -// E.g. ERROR: apport (pid 4557) Sun Jan 1 15:50:45 2015 -timestamp:[A-Z][a-z]{2} [A-Z][a-z]{2} [ \d]\d \d{2}:\d{2}:\d{2} \d{4} - -// === // Delimiters -// === +delimiters: \t\r\n!"#$%&'\(\)\*,:;<>?@\[\]\^_`\{\|\}~ -delimiters: \t\r\n!"#$%&'\(\)\*,:;\<=\>\?@\[\]\^_`\{\|\}~ +// Timestamps (using the `timestamp` keyword) +// E.g. 2015-01-31 15:50:45,392 +// E.g. 2015-01-31 15:50:45.392 +// E.g. 2015-01-31 15:50:45 +timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3}){0,1} +// E.g. [20150131-15:50:45] +timestamp:\[\d{8}\-\d{2}:\d{2}:\d{2}\] -// === // Specially-encoded variables (using the `int` and `float` keywords) -// === - int:\-{0,1}[0-9]+ float:\-{0,1}[0-9]+\.[0-9]+ -// === // Dictionary variables -// === - hex:[a-fA-F]+ - -// NOTE: `equals` and `hasNumber` only work if `=` is a delimiter, due to -// https://github.com/y-scope/log-surgeon/issues/131 -equals:.*=(?.*[a-zA-Z0-9].*) hasNumber:.*\d.* +equals:.*=.*[a-zA-Z0-9].*