diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt index 920e6bcbcb..457a3c2473 100644 --- a/components/core/config/schemas.txt +++ b/components/core/config/schemas.txt @@ -1,19 +1,19 @@ // Delimiters delimiters: \t\r\n!"#$%&'()*,:;<>?@[]^_`{}|~ -// Timestamps (using the `timestamp` keyword) +// Headers (using the `timestamp` capture keyword) // E.g. 2015-01-31 15:50:45,392 // E.g. 2015-01-31 15:50:45.392 // E.g. 2015-01-31 15:50:45 -timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3}){0,1} +header:(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3}){0,1}) // E.g. [20150131-15:50:45] -timestamp:\[\d{8}\-\d{2}:\d{2}:\d{2}\] +header:(?\[\d{8}-\d{2}:\d{2}:\d{2}\]) // Specially-encoded variables (using the `int` and `float` keywords) -int:\-{0,1}[0-9]+ -float:\-{0,1}[0-9]+\.[0-9]+ +int:-?\d+ +float:-?\d+\.\d+ // Dictionary variables hex:[a-fA-F]+ hasNumber:.*\d.* -equals:[a-zA-Z0-9]+=(?.*[a-zA-Z0-9].*) +equals:.*=.*[a-zA-Z0-9].* diff --git a/components/core/src/clp/Utils.cpp b/components/core/src/clp/Utils.cpp index 382215cf44..d8a080b3db 100644 --- a/components/core/src/clp/Utils.cpp +++ b/components/core/src/clp/Utils.cpp @@ -1,31 +1,26 @@ #include "Utils.hpp" #include -#include #include #include -#include #include -#include #include -#include -#include #include #include #include -#include #include "spdlog_with_specializations.hpp" -using std::list; +namespace clp { +using log_surgeon::finite_automata::ByteNfaState; +using log_surgeon::finite_automata::RegexASTLiteral; using std::make_unique; using std::string; using std::unique_ptr; using std::vector; -namespace clp { ErrorCode create_directory(string const& path, mode_t mode, bool exist_ok) { int retval = mkdir(path.c_str(), mode); if (0 != retval) { @@ -135,9 +130,8 @@ load_lexer_from_file(std::string const& schema_file_path, log_surgeon::lexers::B lexer.m_symbol_id[log_surgeon::cTokenEnd] = static_cast(log_surgeon::SymbolId::TokenEnd); lexer.m_symbol_id[log_surgeon::cTokenUncaughtString] = static_cast(log_surgeon::SymbolId::TokenUncaughtString); - // cTokenInt, cTokenFloat, cTokenFirstTimestamp, and cTokenNewlineTimestamp each have unknown - // rule(s) until specified by the user so can't be explicitly added and are done by looping over - // schema_vars (user schema) + // cTokenInt, cTokenFloat, and cTokenHeader each have unknown rule(s) until specified by the + // user so can't be explicitly added and are done by looping over schema_vars (user schema) lexer.m_symbol_id[log_surgeon::cTokenInt] = static_cast(log_surgeon::SymbolId::TokenInt); lexer.m_symbol_id[log_surgeon::cTokenFloat] = static_cast(log_surgeon::SymbolId::TokenFloat); @@ -159,15 +153,7 @@ load_lexer_from_file(std::string const& schema_file_path, log_surgeon::lexers::B lexer.m_id_symbol[static_cast(log_surgeon::SymbolId::TokenNewline)] = log_surgeon::cTokenNewline; - lexer.add_rule( - lexer.m_symbol_id["newLine"], - std::move( - std::make_unique>(log_surgeon::finite_automata:: - RegexASTLiteral('\n')) - ) - ); + lexer.add_rule(lexer.m_symbol_id["newLine"], make_unique>('\n')); for (auto const& delimiters_ast : schema_ast->m_delimiters) { auto* delimiters_ptr = dynamic_cast(delimiters_ast.get()); @@ -185,7 +171,12 @@ load_lexer_from_file(std::string const& schema_file_path, log_surgeon::lexers::B auto* rule = dynamic_cast(parser_ast.get()); // Capture groups are temporarily disabled, until NFA intersection supports for search. - auto const num_captures{rule->m_regex_ptr->get_subtree_positive_captures().size()}; + auto const& captures{rule->m_regex_ptr->get_subtree_positive_captures()}; + auto const num_captures{captures.size()}; + if ("header" == rule->m_name && 1 == num_captures && "timestamp" == captures[0]->get_name()) + { + continue; + } if (0 < num_captures) { throw std::runtime_error( schema_file_path + ":" + std::to_string(rule->m_line_num + 1) @@ -195,55 +186,13 @@ load_lexer_from_file(std::string const& schema_file_path, log_surgeon::lexers::B ); } - if ("timestamp" == rule->m_name) { - continue; - } + // transform '.' from any-character into any non-delimiter character + rule->m_regex_ptr->remove_delimiters_from_wildcard(delimiters); if (lexer.m_symbol_id.find(rule->m_name) == lexer.m_symbol_id.end()) { lexer.m_symbol_id[rule->m_name] = lexer.m_symbol_id.size(); lexer.m_id_symbol[lexer.m_symbol_id[rule->m_name]] = rule->m_name; } - - // transform '.' from any-character into any non-delimiter character - rule->m_regex_ptr->remove_delimiters_from_wildcard(delimiters); - - std::array is_possible_input{}; - rule->m_regex_ptr->set_possible_inputs_to_true(is_possible_input); - bool contains_delimiter = false; - uint32_t delimiter_name; - for (uint32_t delimiter : delimiters) { - if (is_possible_input[delimiter]) { - contains_delimiter = true; - delimiter_name = delimiter; - break; - } - } - - if (contains_delimiter) { - FileReader schema_reader{schema_ast->m_file_path}; - // more detailed debugging based on looking at the file - string line; - for (uint32_t i = 0; i <= rule->m_line_num; i++) { - schema_reader.read_to_delimiter('\n', false, false, line); - } - int colon_pos = 0; - for (char i : line) { - colon_pos++; - if (i == ':') { - break; - } - } - string indent(10, ' '); - string spaces(colon_pos, ' '); - string arrows(line.size() - colon_pos, '^'); - - throw std::runtime_error( - schema_file_path + ":" + std::to_string(rule->m_line_num + 1) + ": error: '" - + rule->m_name + "' has regex pattern which contains delimiter '" - + char(delimiter_name) + "'.\n" + indent + line + "\n" + indent + spaces - + arrows + "\n" - ); - } lexer.add_rule(lexer.m_symbol_id[rule->m_name], std::move(rule->m_regex_ptr)); } lexer.generate(); diff --git a/components/core/src/clp/clp/run.cpp b/components/core/src/clp/clp/run.cpp index 09e3475c94..e1e73bf11f 100644 --- a/components/core/src/clp/clp/run.cpp +++ b/components/core/src/clp/clp/run.cpp @@ -64,12 +64,26 @@ int run(int argc, char const* argv[]) { // Capture groups are temporarily disabled, until NFA intersection support for search. auto const& lexer{reader_parser->get_log_parser().m_lexer}; for (auto const& [rule_id, rule_name] : lexer.m_id_symbol) { - if (lexer.get_captures_from_rule_id(rule_id).has_value()) { - throw std::runtime_error( - schema_file_path + ": error: the schema rule '" + rule_name - + "' has a regex pattern containing capture groups.\n" - ); + auto optional_captures{lexer.get_captures_from_rule_id(rule_id)}; + if (false == optional_captures.has_value()) { + continue; } + + auto const& captures{optional_captures.value()}; + if (captures.empty()) { + continue; + } + + if ("header" == rule_name && 1 == captures.size() + && "timestamp" == captures[0]->get_name()) + { + continue; + } + + throw std::runtime_error( + schema_file_path + ": error: the schema rule '" + rule_name + + "' has a regex pattern containing capture groups.\n" + ); } } diff --git a/components/core/tests/test-ParserWithUserSchema.cpp b/components/core/tests/test-ParserWithUserSchema.cpp index a74de6d8a3..2ae28525c4 100644 --- a/components/core/tests/test-ParserWithUserSchema.cpp +++ b/components/core/tests/test-ParserWithUserSchema.cpp @@ -20,9 +20,7 @@ #include #include #include -#include #include -#include #include #include "TestOutputCleaner.hpp" @@ -85,6 +83,10 @@ auto run_clp_compress( input_path_str.data(), nullptr }; + // `clp::clp::run` registers a logger for `spdlog` that persists across runs. `spdlog` will + // error if a logger with the same name already exists. `spdlog::drop_all` clears all loggers, + // ensuring `clp::clp::run` can safely create a fresh logger for each new call. + spdlog::drop_all(); return clp::clp::run(static_cast(argv.size() - 1), argv.data()); } } // namespace @@ -191,7 +193,7 @@ TEST_CASE("Test lexer", "[Search]") { } } -TEST_CASE("Error on schema rule with a single capture group", "[load_lexer]") { +TEST_CASE("Error on schema rule with a single non-header capture group", "[load_lexer]") { auto const schema_file_path{get_test_schema_files_dir() / "single_capture_group.txt"}; ByteLexer lexer; REQUIRE_THROWS_WITH( @@ -202,7 +204,7 @@ TEST_CASE("Error on schema rule with a single capture group", "[load_lexer]") { ); } -TEST_CASE("Error on schema rule with multiple capture groups", "[load_lexer]") { +TEST_CASE("Error on schema rule with multiple non-header capture groups", "[load_lexer]") { auto const schema_file_path{get_test_schema_files_dir() / "multiple_capture_groups.txt"}; ByteLexer lexer; REQUIRE_THROWS_WITH( @@ -213,7 +215,7 @@ TEST_CASE("Error on schema rule with multiple capture groups", "[load_lexer]") { ); } -TEST_CASE("Verify CLP compression fails with capture groups", "[Compression]") { +TEST_CASE("Verify CLP compression fails with non-header capture groups", "[Compression]") { auto const log_file_path{get_test_log_dir() / "log_with_capture.txt"}; auto const schema_file_path{get_test_schema_files_dir() / "single_capture_group.txt"}; TestOutputCleaner const cleaner{{std::string{cTestArchiveDirectory}}}; @@ -226,3 +228,83 @@ TEST_CASE("Verify CLP compression fails with capture groups", "[Compression]") { "groups.\n" ); } + +TEST_CASE("Succeed on header rule with no capture", "[load_lexer]") { + auto const schema_file_path{get_test_schema_files_dir() / "header_with_no_capture.txt"}; + ByteLexer lexer; + REQUIRE_NOTHROW(load_lexer_from_file(schema_file_path, lexer)); +} + +TEST_CASE("Succeed on header rule with a single timestamp capture", "[load_lexer]") { + auto const schema_file_path{get_test_schema_files_dir() / "header_with_timestamp.txt"}; + ByteLexer lexer; + REQUIRE_NOTHROW(load_lexer_from_file(schema_file_path, lexer)); +} + +TEST_CASE("Error on header rule with a single non-timestamp capture", "[load_lexer]") { + auto const schema_file_path{get_test_schema_files_dir() / "header_with_int.txt"}; + ByteLexer lexer; + REQUIRE_THROWS_WITH( + load_lexer_from_file(schema_file_path, lexer), + schema_file_path.string() + + ":3: error: the schema rule 'header' has a regex pattern containing capture " + "groups (found 1).\n" + ); +} + +TEST_CASE("Error on header rule with a timestamp and non-timestamp capture", "[load_lexer]") { + auto const schema_file_path{get_test_schema_files_dir() / "header_with_timestamp_and_int.txt"}; + ByteLexer lexer; + REQUIRE_THROWS_WITH( + load_lexer_from_file(schema_file_path, lexer), + schema_file_path.string() + + ":3: error: the schema rule 'header' has a regex pattern containing capture " + "groups (found 2).\n" + ); +} + +TEST_CASE("Verify CLP compression succeeds with non-capture header", "[Compression]") { + auto const log_file_path{get_test_log_dir() / "log_with_capture.txt"}; + auto const schema_file_path{get_test_schema_files_dir() / "header_with_no_capture.txt"}; + TestOutputCleaner const cleaner{{std::string{cTestArchiveDirectory}}}; + std::filesystem::create_directory(cTestArchiveDirectory); + + REQUIRE(0 == run_clp_compress(schema_file_path, cTestArchiveDirectory, log_file_path)); +} + +TEST_CASE("Verify CLP compression succeeds with timestamp capture header", "[Compression]") { + auto const log_file_path{get_test_log_dir() / "log_with_capture.txt"}; + auto const schema_file_path{get_test_schema_files_dir() / "header_with_timestamp.txt"}; + TestOutputCleaner const cleaner{{std::string{cTestArchiveDirectory}}}; + std::filesystem::create_directory(cTestArchiveDirectory); + + REQUIRE(0 == run_clp_compress(schema_file_path, cTestArchiveDirectory, log_file_path)); +} + +TEST_CASE("Verify CLP compression fails with non-timestamp capture header", "[Compression]") { + auto const log_file_path{get_test_log_dir() / "log_with_capture.txt"}; + auto const schema_file_path{get_test_schema_files_dir() / "header_with_int.txt"}; + TestOutputCleaner const cleaner{{std::string{cTestArchiveDirectory}}}; + std::filesystem::create_directory(cTestArchiveDirectory); + + REQUIRE_THROWS_WITH( + run_clp_compress(schema_file_path, cTestArchiveDirectory, log_file_path), + schema_file_path.string() + + ": error: the schema rule 'header' has a regex pattern containing capture " + "groups.\n" + ); +} + +TEST_CASE("Verify CLP compression fails with multi-capture header", "[Compression]") { + auto const log_file_path{get_test_log_dir() / "log_with_capture.txt"}; + auto const schema_file_path{get_test_schema_files_dir() / "header_with_timestamp_and_int.txt"}; + TestOutputCleaner const cleaner{{std::string{cTestArchiveDirectory}}}; + std::filesystem::create_directory(cTestArchiveDirectory); + + REQUIRE_THROWS_WITH( + run_clp_compress(schema_file_path, cTestArchiveDirectory, log_file_path), + schema_file_path.string() + + ": error: the schema rule 'header' has a regex pattern containing capture " + "groups.\n" + ); +} diff --git a/components/core/tests/test_schema_files/header_with_int.txt b/components/core/tests/test_schema_files/header_with_int.txt new file mode 100644 index 0000000000..5a0214346b --- /dev/null +++ b/components/core/tests/test_schema_files/header_with_int.txt @@ -0,0 +1,3 @@ +delimiters: \r\n + +header:(?\d+) diff --git a/components/core/tests/test_schema_files/header_with_no_capture.txt b/components/core/tests/test_schema_files/header_with_no_capture.txt new file mode 100644 index 0000000000..56c4c3b149 --- /dev/null +++ b/components/core/tests/test_schema_files/header_with_no_capture.txt @@ -0,0 +1,3 @@ +delimiters: \r\n + +header:\d{2}-\d{2}-\d{4}:\d{2}:\d{2}:\d{2}\.\d{3} diff --git a/components/core/tests/test_schema_files/header_with_timestamp.txt b/components/core/tests/test_schema_files/header_with_timestamp.txt new file mode 100644 index 0000000000..1669fd022a --- /dev/null +++ b/components/core/tests/test_schema_files/header_with_timestamp.txt @@ -0,0 +1,3 @@ +delimiters: \r\n + +header:(?\d{2}-\d{2}-\d{4}:\d{2}:\d{2}:\d{2}\.\d{3}) diff --git a/components/core/tests/test_schema_files/header_with_timestamp_and_int.txt b/components/core/tests/test_schema_files/header_with_timestamp_and_int.txt new file mode 100644 index 0000000000..62dcc5b074 --- /dev/null +++ b/components/core/tests/test_schema_files/header_with_timestamp_and_int.txt @@ -0,0 +1,3 @@ +delimiters: \r\n + +header:(?\d{2}-\d{2}-\d{4}:\d{2}:\d{2}:\d{2}\.\d{3}) and (?\d+)