diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 7c2f408a02..40a615c165 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -579,6 +579,8 @@ set(SOURCE_FILES_unitTest src/clp/GlobalSQLiteMetadataDB.hpp src/clp/Grep.cpp src/clp/Grep.hpp + src/clp/GrepCore.cpp + src/clp/GrepCore.hpp src/clp/hash_utils.cpp src/clp/hash_utils.hpp src/clp/ir/constants.hpp @@ -627,6 +629,8 @@ set(SOURCE_FILES_unitTest src/clp/Profiler.hpp src/clp/Query.cpp src/clp/Query.hpp + src/clp/QueryToken.cpp + src/clp/QueryToken.hpp src/clp/ReaderInterface.cpp src/clp/ReaderInterface.hpp src/clp/ReadOnlyMemoryMappedFile.cpp @@ -716,7 +720,7 @@ set(SOURCE_FILES_unitTest tests/test-ffi_KeyValuePairLogEvent.cpp tests/test-ffi_SchemaTree.cpp tests/test-FileDescriptorReader.cpp - tests/test-Grep.cpp + tests/test-GrepCore.cpp tests/test-hash_utils.cpp tests/test-ir_encoding_methods.cpp tests/test-ir_parsing.cpp diff --git a/components/core/src/clp/EncodedVariableInterpreter.hpp b/components/core/src/clp/EncodedVariableInterpreter.hpp index 9f8fb87441..3c93a2cf4f 100644 --- a/components/core/src/clp/EncodedVariableInterpreter.hpp +++ b/components/core/src/clp/EncodedVariableInterpreter.hpp @@ -184,7 +184,6 @@ class EncodedVariableInterpreter { * Encodes a string-form variable, and if it is dictionary variable, searches for its ID in the * given variable dictionary. * @tparam VariableDictionaryReaderType - * @tparam VariableDictionaryEntryType * @param var_str * @param var_dict * @param ignore_case @@ -194,9 +193,7 @@ class EncodedVariableInterpreter { * dictionary * @return false otherwise */ - template < - typename VariableDictionaryReaderType, - typename VariableDictionaryEntryType = typename VariableDictionaryReaderType::entry_t> + template static bool encode_and_search_dictionary( std::string_view var_str, VariableDictionaryReaderType const& var_dict, @@ -437,7 +434,7 @@ bool EncodedVariableInterpreter::decode_variables_into_message( return true; } -template +template bool EncodedVariableInterpreter::encode_and_search_dictionary( std::string_view var_str, VariableDictionaryReaderType const& var_dict, @@ -468,20 +465,18 @@ bool EncodedVariableInterpreter::encode_and_search_dictionary( if (entries.size() == 1) { auto const* entry = entries.at(0); - sub_query.add_dict_var(encode_var_dict_id(entry->get_id()), entry); + sub_query.add_dict_var(encode_var_dict_id(entry->get_id()), entry->get_id()); return true; } - std::unordered_set const entries_set{ - entries.cbegin(), - entries.cend() - }; std::unordered_set encoded_vars; + std::unordered_set var_dict_ids; encoded_vars.reserve(entries.size()); for (auto const* entry : entries) { encoded_vars.emplace(encode_var_dict_id(entry->get_id())); + var_dict_ids.emplace(entry->get_id()); } - sub_query.add_imprecise_dict_var(encoded_vars, entries_set); + sub_query.add_imprecise_dict_var(encoded_vars, var_dict_ids); } return true; @@ -504,11 +499,13 @@ bool EncodedVariableInterpreter::wildcard_search_dictionary_and_get_encoded_matc // Encode matches std::unordered_set encoded_vars; + std::unordered_set var_dict_ids; for (auto entry : var_dict_entries) { encoded_vars.emplace(encode_var_dict_id(entry->get_id())); + var_dict_ids.emplace(entry->get_id()); } - sub_query.add_imprecise_dict_var(encoded_vars, var_dict_entries); + sub_query.add_imprecise_dict_var(encoded_vars, var_dict_ids); return true; } diff --git a/components/core/src/clp/Grep.cpp b/components/core/src/clp/Grep.cpp index a8decb7fb3..e8811a791d 100644 --- a/components/core/src/clp/Grep.cpp +++ b/components/core/src/clp/Grep.cpp @@ -1,282 +1,23 @@ #include "Grep.hpp" -#include +#include +#include -#include #include -#include "EncodedVariableInterpreter.hpp" -#include "ir/parsing.hpp" -#include "ir/types.hpp" -#include "LogSurgeonReader.hpp" -#include "StringReader.hpp" -#include "Utils.hpp" +#include "streaming_archive/reader/Archive.hpp" +#include "streaming_archive/reader/File.hpp" +#include "streaming_archive/reader/Message.hpp" -using clp::ir::is_delim; using clp::streaming_archive::reader::Archive; using clp::streaming_archive::reader::File; using clp::streaming_archive::reader::Message; -using clp::string_utils::clean_up_wildcard_search_string; -using clp::string_utils::is_alphabet; -using clp::string_utils::is_wildcard; using clp::string_utils::wildcard_match_unsafe; using std::string; using std::vector; namespace clp { namespace { -// Local types -enum class SubQueryMatchabilityResult { - MayMatch, // The subquery might match a message - WontMatch, // The subquery has no chance of matching a message - SupercedesAllSubQueries // The subquery will cause all messages to be matched -}; - -// Class representing a token in a query. It is used to interpret a token in user's search string. -class QueryToken { -public: - // Constructors - QueryToken(string const& query_string, size_t begin_pos, size_t end_pos, bool is_var); - - // Methods - bool cannot_convert_to_non_dict_var() const; - bool contains_wildcards() const; - bool has_greedy_wildcard_in_middle() const; - bool has_prefix_greedy_wildcard() const; - bool has_suffix_greedy_wildcard() const; - bool is_ambiguous_token() const; - bool is_float_var() const; - bool is_int_var() const; - bool is_var() const; - bool is_wildcard() const; - - size_t get_begin_pos() const; - size_t get_end_pos() const; - string const& get_value() const; - - bool change_to_next_possible_type(); - -private: - // Types - // Type for the purpose of generating different subqueries. E.g., if a token is of type - // DictOrIntVar, it would generate a different subquery than if it was of type Logtype. - enum class Type { - Wildcard, - // Ambiguous indicates the token can be more than one of the types listed below - Ambiguous, - Logtype, - DictionaryVar, - FloatVar, - IntVar - }; - - // Variables - bool m_cannot_convert_to_non_dict_var; - bool m_contains_wildcards; - bool m_has_greedy_wildcard_in_middle; - bool m_has_prefix_greedy_wildcard; - bool m_has_suffix_greedy_wildcard; - - size_t m_begin_pos; - size_t m_end_pos; - string m_value; - - // Type if variable has unambiguous type - Type m_type; - // Types if variable type is ambiguous - vector m_possible_types; - // Index of the current possible type selected for generating a subquery - size_t m_current_possible_type_ix; -}; - -QueryToken::QueryToken( - string const& query_string, - size_t const begin_pos, - size_t const end_pos, - bool const is_var -) - : m_current_possible_type_ix(0) { - m_begin_pos = begin_pos; - m_end_pos = end_pos; - m_value.assign(query_string, m_begin_pos, m_end_pos - m_begin_pos); - - // Set wildcard booleans and determine type - if ("*" == m_value) { - m_has_prefix_greedy_wildcard = true; - m_has_suffix_greedy_wildcard = false; - m_has_greedy_wildcard_in_middle = false; - m_contains_wildcards = true; - m_type = Type::Wildcard; - } else { - m_has_prefix_greedy_wildcard = ('*' == m_value[0]); - m_has_suffix_greedy_wildcard = ('*' == m_value[m_value.length() - 1]); - - m_has_greedy_wildcard_in_middle = false; - for (size_t i = 1; i < m_value.length() - 1; ++i) { - if ('*' == m_value[i]) { - m_has_greedy_wildcard_in_middle = true; - break; - } - } - - m_contains_wildcards - = (m_has_prefix_greedy_wildcard || m_has_suffix_greedy_wildcard - || m_has_greedy_wildcard_in_middle); - - if (!is_var) { - if (!m_contains_wildcards) { - m_type = Type::Logtype; - } else { - m_type = Type::Ambiguous; - m_possible_types.push_back(Type::Logtype); - m_possible_types.push_back(Type::IntVar); - m_possible_types.push_back(Type::FloatVar); - m_possible_types.push_back(Type::DictionaryVar); - } - } else { - string value_without_wildcards = m_value; - if (m_has_prefix_greedy_wildcard) { - value_without_wildcards = value_without_wildcards.substr(1); - } - if (m_has_suffix_greedy_wildcard) { - value_without_wildcards.resize(value_without_wildcards.length() - 1); - } - - encoded_variable_t encoded_var; - bool converts_to_non_dict_var = false; - if (EncodedVariableInterpreter::convert_string_to_representable_integer_var( - value_without_wildcards, - encoded_var - ) - || EncodedVariableInterpreter::convert_string_to_representable_float_var( - value_without_wildcards, - encoded_var - )) - { - converts_to_non_dict_var = true; - } - - if (!converts_to_non_dict_var) { - // Dictionary variable - m_type = Type::DictionaryVar; - m_cannot_convert_to_non_dict_var = true; - } else { - m_type = Type::Ambiguous; - m_possible_types.push_back(Type::IntVar); - m_possible_types.push_back(Type::FloatVar); - m_possible_types.push_back(Type::DictionaryVar); - m_cannot_convert_to_non_dict_var = false; - } - } - } -} - -bool QueryToken::cannot_convert_to_non_dict_var() const { - return m_cannot_convert_to_non_dict_var; -} - -bool QueryToken::contains_wildcards() const { - return m_contains_wildcards; -} - -bool QueryToken::has_greedy_wildcard_in_middle() const { - return m_has_greedy_wildcard_in_middle; -} - -bool QueryToken::has_prefix_greedy_wildcard() const { - return m_has_prefix_greedy_wildcard; -} - -bool QueryToken::has_suffix_greedy_wildcard() const { - return m_has_suffix_greedy_wildcard; -} - -bool QueryToken::is_ambiguous_token() const { - return Type::Ambiguous == m_type; -} - -bool QueryToken::is_float_var() const { - Type type; - if (Type::Ambiguous == m_type) { - type = m_possible_types[m_current_possible_type_ix]; - } else { - type = m_type; - } - return Type::FloatVar == type; -} - -bool QueryToken::is_int_var() const { - Type type; - if (Type::Ambiguous == m_type) { - type = m_possible_types[m_current_possible_type_ix]; - } else { - type = m_type; - } - return Type::IntVar == type; -} - -bool QueryToken::is_var() const { - Type type; - if (Type::Ambiguous == m_type) { - type = m_possible_types[m_current_possible_type_ix]; - } else { - type = m_type; - } - return (Type::IntVar == type || Type::FloatVar == type || Type::DictionaryVar == type); -} - -bool QueryToken::is_wildcard() const { - return Type::Wildcard == m_type; -} - -size_t QueryToken::get_begin_pos() const { - return m_begin_pos; -} - -size_t QueryToken::get_end_pos() const { - return m_end_pos; -} - -string const& QueryToken::get_value() const { - return m_value; -} - -bool QueryToken::change_to_next_possible_type() { - if (m_current_possible_type_ix < m_possible_types.size() - 1) { - ++m_current_possible_type_ix; - return true; - } else { - m_current_possible_type_ix = 0; - return false; - } -} - -/** - * Wraps the tokens returned from the log_surgeon lexer, and stores the variable ids of the tokens - * in a search query in a set. This allows for optimized search performance. - */ -class SearchToken : public log_surgeon::Token { -public: - std::set m_type_ids_set; -}; - -// Local prototypes -/** - * Process a QueryToken that is definitely a variable - * @param query_token - * @param archive - * @param ignore_case - * @param sub_query - * @param logtype - * @return true if this token might match a message, false otherwise - */ -bool process_var_token( - QueryToken const& query_token, - Archive const& archive, - bool ignore_case, - SubQuery& sub_query, - string& logtype -); /** * Finds a message matching the given query * @param query @@ -293,84 +34,6 @@ bool find_matching_message( File& compressed_file, Message& compressed_msg ); -/** - * Generates logtypes and variables for subquery - * @param archive - * @param processed_search_string - * @param query_tokens - * @param ignore_case - * @param sub_query - * @return SubQueryMatchabilityResult::SupercedesAllSubQueries - * @return SubQueryMatchabilityResult::WontMatch - * @return SubQueryMatchabilityResult::MayMatch - */ -SubQueryMatchabilityResult generate_logtypes_and_vars_for_subquery( - Archive const& archive, - string& processed_search_string, - vector& query_tokens, - bool ignore_case, - SubQuery& sub_query -); - -bool process_var_token( - QueryToken const& query_token, - Archive const& archive, - bool ignore_case, - SubQuery& sub_query, - string& logtype -) { - // Even though we may have a precise variable, we still fallback to decompressing to ensure that - // it is in the right place in the message - sub_query.mark_wildcard_match_required(); - - // Create QueryVar corresponding to token - if (!query_token.contains_wildcards()) { - if (EncodedVariableInterpreter::encode_and_search_dictionary( - query_token.get_value(), - archive.get_var_dictionary(), - ignore_case, - logtype, - sub_query - ) - == false) - { - // Variable doesn't exist in dictionary - return false; - } - } else { - if (query_token.has_prefix_greedy_wildcard()) { - logtype += '*'; - } - - if (query_token.is_float_var()) { - EncodedVariableInterpreter::add_float_var(logtype); - } else if (query_token.is_int_var()) { - EncodedVariableInterpreter::add_int_var(logtype); - } else { - EncodedVariableInterpreter::add_dict_var(logtype); - - if (query_token.cannot_convert_to_non_dict_var()) { - // Must be a dictionary variable, so search variable dictionary - if (!EncodedVariableInterpreter::wildcard_search_dictionary_and_get_encoded_matches( - query_token.get_value(), - archive.get_var_dictionary(), - ignore_case, - sub_query - )) - { - // Variable doesn't exist in dictionary - return false; - } - } - } - - if (query_token.has_suffix_greedy_wildcard()) { - logtype += '*'; - } - } - - return true; -} bool find_matching_message( Query const& query, @@ -406,466 +69,8 @@ bool find_matching_message( return true; } - -SubQueryMatchabilityResult generate_logtypes_and_vars_for_subquery( - Archive const& archive, - string& processed_search_string, - vector& query_tokens, - bool ignore_case, - SubQuery& sub_query -) { - size_t last_token_end_pos = 0; - string logtype; - auto escape_handler - = [](std::string_view constant, size_t char_to_escape_pos, string& logtype) -> void { - auto const escape_char{enum_to_underlying_type(ir::VariablePlaceholder::Escape)}; - auto const next_char_pos{char_to_escape_pos + 1}; - // NOTE: We don't want to add additional escapes for wildcards that have been escaped. E.g., - // the query "\\*" should remain unchanged. - if (next_char_pos < constant.length() && false == is_wildcard(constant[next_char_pos])) { - logtype += escape_char; - } else if (ir::is_variable_placeholder(constant[char_to_escape_pos])) { - logtype += escape_char; - logtype += escape_char; - } - }; - for (auto const& query_token : query_tokens) { - // Append from end of last token to beginning of this token, to logtype - ir::append_constant_to_logtype( - static_cast(processed_search_string) - .substr(last_token_end_pos, - query_token.get_begin_pos() - last_token_end_pos), - escape_handler, - logtype - ); - last_token_end_pos = query_token.get_end_pos(); - - if (query_token.is_wildcard()) { - logtype += '*'; - } else if (query_token.has_greedy_wildcard_in_middle()) { - // Fallback to decompression + wildcard matching for now to avoid handling queries where - // the pieces of the token on either side of each wildcard need to be processed as - // ambiguous tokens - sub_query.mark_wildcard_match_required(); - if (!query_token.is_var()) { - logtype += '*'; - } else { - logtype += '*'; - EncodedVariableInterpreter::add_dict_var(logtype); - logtype += '*'; - } - } else { - if (!query_token.is_var()) { - ir::append_constant_to_logtype(query_token.get_value(), escape_handler, logtype); - } else if (!process_var_token(query_token, archive, ignore_case, sub_query, logtype)) { - return SubQueryMatchabilityResult::WontMatch; - } - } - } - - if (last_token_end_pos < processed_search_string.length()) { - // Append from end of last token to end - ir::append_constant_to_logtype( - static_cast(processed_search_string) - .substr(last_token_end_pos, string::npos), - escape_handler, - logtype - ); - last_token_end_pos = processed_search_string.length(); - } - - if ("*" == logtype) { - // Logtype will match all messages - return SubQueryMatchabilityResult::SupercedesAllSubQueries; - } - - // Find matching logtypes - std::unordered_set possible_logtype_entries; - archive.get_logtype_dictionary() - .get_entries_matching_wildcard_string(logtype, ignore_case, possible_logtype_entries); - if (possible_logtype_entries.empty()) { - return SubQueryMatchabilityResult::WontMatch; - } - sub_query.set_possible_logtypes(possible_logtype_entries); - - // Calculate the IDs of the segments that may contain results for the sub-query now that we've - // calculated the matching logtypes and variables - sub_query.calculate_ids_of_matching_segments(); - - return SubQueryMatchabilityResult::MayMatch; -} } // namespace -std::optional Grep::process_raw_query( - Archive const& archive, - string const& search_string, - epochtime_t search_begin_ts, - epochtime_t search_end_ts, - bool ignore_case, - log_surgeon::lexers::ByteLexer& lexer, - bool use_heuristic -) { - // Add prefix and suffix '*' to make the search a sub-string match - string processed_search_string = "*"; - processed_search_string += search_string; - processed_search_string += '*'; - processed_search_string = clean_up_wildcard_search_string(processed_search_string); - - // Split search_string into tokens with wildcards - vector query_tokens; - size_t begin_pos = 0; - size_t end_pos = 0; - bool is_var; - string search_string_for_sub_queries{processed_search_string}; - if (use_heuristic) { - // Replace '?' wildcards with '*' wildcards since we currently have no support for - // generating sub-queries with '?' wildcards. The final wildcard match on the decompressed - // message uses the original wildcards, so correctness will be maintained. - std::replace( - search_string_for_sub_queries.begin(), - search_string_for_sub_queries.end(), - '?', - '*' - ); - // Clean-up in case any instances of "?*" or "*?" were changed into "**" - search_string_for_sub_queries - = clean_up_wildcard_search_string(search_string_for_sub_queries); - while (get_bounds_of_next_potential_var( - search_string_for_sub_queries, - begin_pos, - end_pos, - is_var - )) - { - query_tokens.emplace_back(search_string_for_sub_queries, begin_pos, end_pos, is_var); - } - } else { - while (get_bounds_of_next_potential_var( - search_string_for_sub_queries, - begin_pos, - end_pos, - is_var, - lexer - )) - { - query_tokens.emplace_back(search_string_for_sub_queries, begin_pos, end_pos, is_var); - } - } - - // Get pointers to all ambiguous tokens. Exclude tokens with wildcards in the middle since we - // fall-back to decompression + wildcard matching for those. - vector ambiguous_tokens; - for (auto& query_token : query_tokens) { - if (!query_token.has_greedy_wildcard_in_middle() && query_token.is_ambiguous_token()) { - ambiguous_tokens.push_back(&query_token); - } - } - - // Generate a sub-query for each combination of ambiguous tokens - // E.g., if there are two ambiguous tokens each of which could be a logtype or variable, we need - // to create: - // - (token1 as logtype) (token2 as logtype) - // - (token1 as logtype) (token2 as var) - // - (token1 as var) (token2 as logtype) - // - (token1 as var) (token2 as var) - vector sub_queries; - string logtype; - bool type_of_one_token_changed = true; - while (type_of_one_token_changed) { - SubQuery sub_query; - - // Compute logtypes and variables for query - auto matchability = generate_logtypes_and_vars_for_subquery( - archive, - search_string_for_sub_queries, - query_tokens, - ignore_case, - sub_query - ); - switch (matchability) { - case SubQueryMatchabilityResult::SupercedesAllSubQueries: - // Since other sub-queries will be superceded by this one, we can stop processing - // now - return Query{ - search_begin_ts, - search_end_ts, - ignore_case, - processed_search_string, - {} - }; - case SubQueryMatchabilityResult::MayMatch: - sub_queries.push_back(std::move(sub_query)); - break; - case SubQueryMatchabilityResult::WontMatch: - default: - // Do nothing - break; - } - - // Update combination of ambiguous tokens - type_of_one_token_changed = false; - for (auto* ambiguous_token : ambiguous_tokens) { - if (ambiguous_token->change_to_next_possible_type()) { - type_of_one_token_changed = true; - break; - } - } - } - - if (sub_queries.empty()) { - return std::nullopt; - } - - return Query{ - search_begin_ts, - search_end_ts, - ignore_case, - processed_search_string, - std::move(sub_queries) - }; -} - -bool Grep::get_bounds_of_next_potential_var( - string const& value, - size_t& begin_pos, - size_t& end_pos, - bool& is_var -) { - auto const value_length = value.length(); - if (end_pos >= value_length) { - return false; - } - - is_var = false; - bool contains_wildcard = false; - while (false == is_var && false == contains_wildcard && begin_pos < value_length) { - // Start search at end of last token - begin_pos = end_pos; - - // Find next wildcard or non-delimiter - bool is_escaped = false; - for (; begin_pos < value_length; ++begin_pos) { - char c = value[begin_pos]; - - if (is_escaped) { - is_escaped = false; - - if (false == is_delim(c)) { - // Found escaped non-delimiter, so reverse the index to retain the escape - // character - --begin_pos; - break; - } - } else if ('\\' == c) { - // Escape character - is_escaped = true; - } else { - if (is_wildcard(c)) { - contains_wildcard = true; - break; - } - if (false == is_delim(c)) { - break; - } - } - } - - bool contains_decimal_digit = false; - bool contains_alphabet = false; - - // Find next delimiter - is_escaped = false; - end_pos = begin_pos; - for (; end_pos < value_length; ++end_pos) { - char c = value[end_pos]; - - if (is_escaped) { - is_escaped = false; - - if (is_delim(c)) { - // Found escaped delimiter, so reverse the index to retain the escape character - --end_pos; - break; - } - } else if ('\\' == c) { - // Escape character - is_escaped = true; - } else { - if (is_wildcard(c)) { - contains_wildcard = true; - } else if (is_delim(c)) { - // Found delimiter that's not also a wildcard - break; - } - } - - if (string_utils::is_decimal_digit(c)) { - contains_decimal_digit = true; - } else if (is_alphabet(c)) { - contains_alphabet = true; - } - } - - // Treat token as a definite variable if: - // - it contains a decimal digit, or - // - it could be a multi-digit hex value, or - // - it's directly preceded by an equals sign and contains an alphabet without a wildcard - // between the equals sign and the first alphabet of the token - auto variable = static_cast(value).substr(begin_pos, end_pos - begin_pos); - if (contains_decimal_digit || ir::could_be_multi_digit_hex_value(variable)) { - is_var = true; - } else if (begin_pos > 0 && '=' == value[begin_pos - 1] && contains_alphabet) { - // Find first alphabet or wildcard in token - is_escaped = false; - bool found_wildcard_before_alphabet = false; - for (auto i = begin_pos; i < end_pos; ++i) { - auto c = value[i]; - - if (is_escaped) { - is_escaped = false; - - if (is_alphabet(c)) { - break; - } - } else if ('\\' == c) { - // Escape character - is_escaped = true; - } else if (is_wildcard(c)) { - found_wildcard_before_alphabet = true; - break; - } - } - - if (false == found_wildcard_before_alphabet) { - is_var = true; - } - } - } - - return (value_length != begin_pos); -} - -bool Grep::get_bounds_of_next_potential_var( - string const& value, - size_t& begin_pos, - size_t& end_pos, - bool& is_var, - log_surgeon::lexers::ByteLexer& lexer -) { - size_t const value_length = value.length(); - if (end_pos >= value_length) { - return false; - } - - is_var = false; - bool contains_wildcard = false; - while (false == is_var && false == contains_wildcard && begin_pos < value_length) { - // Start search at end of last token - begin_pos = end_pos; - - // Find variable begin or wildcard - bool is_escaped = false; - for (; begin_pos < value_length; ++begin_pos) { - char c = value[begin_pos]; - - if (is_escaped) { - is_escaped = false; - - if (false == lexer.is_delimiter(c)) { - // Found escaped non-delimiter, so reverse the index to retain the escape - // character - --begin_pos; - break; - } - } else if ('\\' == c) { - // Escape character - is_escaped = true; - } else { - if (is_wildcard(c)) { - contains_wildcard = true; - break; - } - if (false == lexer.is_delimiter(c)) { - break; - } - } - } - - // Find next delimiter - is_escaped = false; - end_pos = begin_pos; - for (; end_pos < value_length; ++end_pos) { - char c = value[end_pos]; - - if (is_escaped) { - is_escaped = false; - - if (lexer.is_delimiter(c)) { - // Found escaped delimiter, so reverse the index to retain the escape character - --end_pos; - break; - } - } else if ('\\' == c) { - // Escape character - is_escaped = true; - } else { - if (is_wildcard(c)) { - contains_wildcard = true; - } else if (lexer.is_delimiter(c)) { - // Found delimiter that's not also a wildcard - break; - } - } - } - - if (end_pos > begin_pos) { - bool has_prefix_wildcard = ('*' == value[begin_pos]) || ('?' == value[begin_pos]); - bool has_suffix_wildcard = ('*' == value[end_pos - 1]) || ('?' == value[begin_pos]); - bool has_wildcard_in_middle = false; - for (size_t i = begin_pos + 1; i < end_pos - 1; ++i) { - if (('*' == value[i] || '?' == value[i]) && value[i - 1] != '\\') { - has_wildcard_in_middle = true; - break; - } - } - SearchToken search_token; - if (has_wildcard_in_middle || has_prefix_wildcard) { - // DO NOTHING - } else { - StringReader string_reader; - LogSurgeonReader reader_wrapper(string_reader); - log_surgeon::ParserInputBuffer parser_input_buffer; - if (has_suffix_wildcard) { // text* - // TODO: creating a string reader, setting it equal to a string, to read it into - // the ParserInputBuffer, seems like a convoluted way to set a string equal to a - // string, should be improved when adding a SearchParser to log_surgeon - string_reader.open(value.substr(begin_pos, end_pos - begin_pos - 1)); - parser_input_buffer.read_if_safe(reader_wrapper); - lexer.reset(); - lexer.scan_with_wildcard(parser_input_buffer, value[end_pos - 1], search_token); - } else { // no wildcards - string_reader.open(value.substr(begin_pos, end_pos - begin_pos)); - parser_input_buffer.read_if_safe(reader_wrapper); - lexer.reset(); - auto [err, token] = lexer.scan(parser_input_buffer); - if (log_surgeon::ErrorCode::Success != err) { - return false; - } - search_token = SearchToken{token.value()}; - search_token.m_type_ids_set.insert(search_token.m_type_ids_ptr->at(0)); - } - auto const& type = search_token.m_type_ids_ptr->at(0); - if (type != static_cast(log_surgeon::SymbolId::TokenUncaughtString) - && type != static_cast(log_surgeon::SymbolId::TokenEnd)) - { - is_var = true; - } - } - } - } - return (value_length != begin_pos); -} - void Grep::calculate_sub_queries_relevant_to_file(File const& compressed_file, vector& queries) { for (auto& query : queries) { diff --git a/components/core/src/clp/Grep.hpp b/components/core/src/clp/Grep.hpp index d4e965c7ee..6a965b6b44 100644 --- a/components/core/src/clp/Grep.hpp +++ b/components/core/src/clp/Grep.hpp @@ -1,11 +1,8 @@ #ifndef CLP_GREP_HPP #define CLP_GREP_HPP -#include #include -#include - #include "Defs.h" #include "Query.hpp" #include "streaming_archive/reader/Archive.hpp" @@ -30,60 +27,6 @@ class Grep { ); // Methods - /** - * Processes a raw user query into a Query - * @param archive - * @param search_string - * @param search_begin_ts - * @param search_end_ts - * @param ignore_case - * @param lexer DFA for determining if input is in the schema - * @param use_heuristic - * @return Query if it may match a message, std::nullopt otherwise - */ - static std::optional process_raw_query( - streaming_archive::reader::Archive const& archive, - std::string const& search_string, - epochtime_t search_begin_ts, - epochtime_t search_end_ts, - bool ignore_case, - log_surgeon::lexers::ByteLexer& lexer, - bool use_heuristic - ); - - /** - * Returns bounds of next potential variable (either a definite variable or a token with - * wildcards) - * @param value String containing token - * @param begin_pos Begin position of last token, changes to begin position of next token - * @param end_pos End position of last token, changes to end position of next token - * @param is_var Whether the token is definitely a variable - * @return true if another potential variable was found, false otherwise - */ - static bool get_bounds_of_next_potential_var( - std::string const& value, - size_t& begin_pos, - size_t& end_pos, - bool& is_var - ); - - /** - * Returns bounds of next potential variable (either a definite variable or a token with - * wildcards) - * @param value String containing token - * @param begin_pos Begin position of last token, changes to begin position of next token - * @param end_pos End position of last token, changes to end position of next token - * @param is_var Whether the token is definitely a variable - * @param lexer DFA for determining if input is in the schema - * @return true if another potential variable was found, false otherwise - */ - static bool get_bounds_of_next_potential_var( - std::string const& value, - size_t& begin_pos, - size_t& end_pos, - bool& is_var, - log_surgeon::lexers::ByteLexer& lexer - ); /** * Marks which sub-queries in each query are relevant to the given file * @param compressed_file diff --git a/components/core/src/clp/GrepCore.cpp b/components/core/src/clp/GrepCore.cpp new file mode 100644 index 0000000000..1a4bf499e2 --- /dev/null +++ b/components/core/src/clp/GrepCore.cpp @@ -0,0 +1,273 @@ +#include "GrepCore.hpp" + +#include +#include +#include +#include + +#include +#include + +#include "ir/parsing.hpp" +#include "LogSurgeonReader.hpp" +#include "QueryToken.hpp" +#include "StringReader.hpp" + +using clp::ir::is_delim; +using clp::string_utils::is_alphabet; +using clp::string_utils::is_wildcard; +using std::string; + +namespace clp { +namespace { +/** + * Wraps the tokens returned from the log_surgeon lexer, and stores the variable ids of the tokens + * in a search query in a set. This allows for optimized search performance. + */ +class SearchToken : public log_surgeon::Token { +public: + std::set m_type_ids_set; +}; +} // namespace + +bool GrepCore::get_bounds_of_next_potential_var( + string const& value, + size_t& begin_pos, + size_t& end_pos, + bool& is_var +) { + auto const value_length = value.length(); + if (end_pos >= value_length) { + return false; + } + + is_var = false; + bool contains_wildcard = false; + while (false == is_var && false == contains_wildcard && begin_pos < value_length) { + // Start search at end of last token + begin_pos = end_pos; + + // Find next wildcard or non-delimiter + bool is_escaped = false; + for (; begin_pos < value_length; ++begin_pos) { + char c = value[begin_pos]; + + if (is_escaped) { + is_escaped = false; + + if (false == is_delim(c)) { + // Found escaped non-delimiter, so reverse the index to retain the escape + // character + --begin_pos; + break; + } + } else if ('\\' == c) { + // Escape character + is_escaped = true; + } else { + if (is_wildcard(c)) { + contains_wildcard = true; + break; + } + if (false == is_delim(c)) { + break; + } + } + } + + bool contains_decimal_digit = false; + bool contains_alphabet = false; + + // Find next delimiter + is_escaped = false; + end_pos = begin_pos; + for (; end_pos < value_length; ++end_pos) { + char c = value[end_pos]; + + if (is_escaped) { + is_escaped = false; + + if (is_delim(c)) { + // Found escaped delimiter, so reverse the index to retain the escape character + --end_pos; + break; + } + } else if ('\\' == c) { + // Escape character + is_escaped = true; + } else { + if (is_wildcard(c)) { + contains_wildcard = true; + } else if (is_delim(c)) { + // Found delimiter that's not also a wildcard + break; + } + } + + if (string_utils::is_decimal_digit(c)) { + contains_decimal_digit = true; + } else if (is_alphabet(c)) { + contains_alphabet = true; + } + } + + // Treat token as a definite variable if: + // - it contains a decimal digit, or + // - it could be a multi-digit hex value, or + // - it's directly preceded by an equals sign and contains an alphabet without a wildcard + // between the equals sign and the first alphabet of the token + auto variable = static_cast(value).substr(begin_pos, end_pos - begin_pos); + if (contains_decimal_digit || ir::could_be_multi_digit_hex_value(variable)) { + is_var = true; + } else if (begin_pos > 0 && '=' == value[begin_pos - 1] && contains_alphabet) { + // Find first alphabet or wildcard in token + is_escaped = false; + bool found_wildcard_before_alphabet = false; + for (auto i = begin_pos; i < end_pos; ++i) { + auto c = value[i]; + + if (is_escaped) { + is_escaped = false; + + if (is_alphabet(c)) { + break; + } + } else if ('\\' == c) { + // Escape character + is_escaped = true; + } else if (is_wildcard(c)) { + found_wildcard_before_alphabet = true; + break; + } + } + + if (false == found_wildcard_before_alphabet) { + is_var = true; + } + } + } + + return (value_length != begin_pos); +} + +bool GrepCore::get_bounds_of_next_potential_var( + string const& value, + size_t& begin_pos, + size_t& end_pos, + bool& is_var, + log_surgeon::lexers::ByteLexer& lexer +) { + size_t const value_length = value.length(); + if (end_pos >= value_length) { + return false; + } + + is_var = false; + bool contains_wildcard = false; + while (false == is_var && false == contains_wildcard && begin_pos < value_length) { + // Start search at end of last token + begin_pos = end_pos; + + // Find variable begin or wildcard + bool is_escaped = false; + for (; begin_pos < value_length; ++begin_pos) { + char c = value[begin_pos]; + + if (is_escaped) { + is_escaped = false; + + if (false == lexer.is_delimiter(c)) { + // Found escaped non-delimiter, so reverse the index to retain the escape + // character + --begin_pos; + break; + } + } else if ('\\' == c) { + // Escape character + is_escaped = true; + } else { + if (is_wildcard(c)) { + contains_wildcard = true; + break; + } + if (false == lexer.is_delimiter(c)) { + break; + } + } + } + + // Find next delimiter + is_escaped = false; + end_pos = begin_pos; + for (; end_pos < value_length; ++end_pos) { + char c = value[end_pos]; + + if (is_escaped) { + is_escaped = false; + + if (lexer.is_delimiter(c)) { + // Found escaped delimiter, so reverse the index to retain the escape character + --end_pos; + break; + } + } else if ('\\' == c) { + // Escape character + is_escaped = true; + } else { + if (is_wildcard(c)) { + contains_wildcard = true; + } else if (lexer.is_delimiter(c)) { + // Found delimiter that's not also a wildcard + break; + } + } + } + + if (end_pos > begin_pos) { + bool has_prefix_wildcard = ('*' == value[begin_pos]) || ('?' == value[begin_pos]); + bool has_suffix_wildcard = ('*' == value[end_pos - 1]) || ('?' == value[end_pos - 1]); + bool has_wildcard_in_middle = false; + for (size_t i = begin_pos + 1; i < end_pos - 1; ++i) { + if (('*' == value[i] || '?' == value[i]) && value[i - 1] != '\\') { + has_wildcard_in_middle = true; + break; + } + } + SearchToken search_token; + if (has_wildcard_in_middle || has_prefix_wildcard) { + // DO NOTHING + } else { + StringReader string_reader; + LogSurgeonReader reader_wrapper(string_reader); + log_surgeon::ParserInputBuffer parser_input_buffer; + if (has_suffix_wildcard) { // text* + // TODO: creating a string reader, setting it equal to a string, to read it into + // the ParserInputBuffer, seems like a convoluted way to set a string equal to a + // string, should be improved when adding a SearchParser to log_surgeon + string_reader.open(value.substr(begin_pos, end_pos - begin_pos - 1)); + parser_input_buffer.read_if_safe(reader_wrapper); + lexer.reset(); + lexer.scan_with_wildcard(parser_input_buffer, value[end_pos - 1], search_token); + } else { // no wildcards + string_reader.open(value.substr(begin_pos, end_pos - begin_pos)); + parser_input_buffer.read_if_safe(reader_wrapper); + lexer.reset(); + auto [err, token] = lexer.scan(parser_input_buffer); + if (log_surgeon::ErrorCode::Success != err) { + return false; + } + search_token = SearchToken{token.value()}; + search_token.m_type_ids_set.insert(search_token.m_type_ids_ptr->at(0)); + } + auto const& type = search_token.m_type_ids_ptr->at(0); + if (type != static_cast(log_surgeon::SymbolId::TokenUncaughtString) + && type != static_cast(log_surgeon::SymbolId::TokenEnd)) + { + is_var = true; + } + } + } + } + return (value_length != begin_pos); +} +} // namespace clp diff --git a/components/core/src/clp/GrepCore.hpp b/components/core/src/clp/GrepCore.hpp new file mode 100644 index 0000000000..1e64a91dc7 --- /dev/null +++ b/components/core/src/clp/GrepCore.hpp @@ -0,0 +1,425 @@ +#ifndef CLP_GREPCORE_HPP +#define CLP_GREPCORE_HPP + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "Defs.h" +#include "EncodedVariableInterpreter.hpp" +#include "ir/parsing.hpp" +#include "ir/types.hpp" +#include "Query.hpp" +#include "QueryToken.hpp" + +namespace clp { +class GrepCore { +public: + // Methods + /** + * Processes a raw user query into a Query. + * + * NOTE: Callers are responsible for ensuring that the search string is "clean", where clean is + * defined by the criteria in `clean_up_wildcard_search_string`. + * + * @tparam LogTypeDictionaryReaderType + * @tparam VariableDictionaryReaderType + * @param logtype_dict + * @param var_dict + * @param search_string + * @param search_begin_ts + * @param search_end_ts + * @param ignore_case + * @param lexer DFA for determining if input is in the schema + * @param use_heuristic + * @return Query if it may match a message, std::nullopt otherwise + */ + template + static std::optional process_raw_query( + LogTypeDictionaryReaderType const& logtype_dict, + VariableDictionaryReaderType const& var_dict, + std::string const& search_string, + epochtime_t search_begin_ts, + epochtime_t search_end_ts, + bool ignore_case, + log_surgeon::lexers::ByteLexer& lexer, + bool use_heuristic + ); + + /** + * Returns bounds of next potential variable (either a definite variable or a token with + * wildcards) + * @param value String containing token + * @param begin_pos Begin position of last token, changes to begin position of next token + * @param end_pos End position of last token, changes to end position of next token + * @param is_var Whether the token is definitely a variable + * @return true if another potential variable was found, false otherwise + */ + static bool get_bounds_of_next_potential_var( + std::string const& value, + size_t& begin_pos, + size_t& end_pos, + bool& is_var + ); + + /** + * Returns bounds of next potential variable (either a definite variable or a token with + * wildcards) + * @param value String containing token + * @param begin_pos Begin position of last token, changes to begin position of next token + * @param end_pos End position of last token, changes to end position of next token + * @param is_var Whether the token is definitely a variable + * @param lexer DFA for determining if input is in the schema + * @return true if another potential variable was found, false otherwise + */ + static bool get_bounds_of_next_potential_var( + std::string const& value, + size_t& begin_pos, + size_t& end_pos, + bool& is_var, + log_surgeon::lexers::ByteLexer& lexer + ); + +private: + // Types + enum class SubQueryMatchabilityResult : uint8_t { + MayMatch, // The subquery might match a message + WontMatch, // The subquery has no chance of matching a message + SupercedesAllSubQueries // The subquery will cause all messages to be matched + }; + + // Methods + /** + * Process a QueryToken that is definitely a variable. + * @tparam VariableDictionaryReaderType + * @param query_token + * @param var_dict + * @param ignore_case + * @param sub_query + * @param logtype + * @return true if this token might match a message, false otherwise + */ + template + static bool process_var_token( + QueryToken const& query_token, + VariableDictionaryReaderType const& var_dict, + bool ignore_case, + SubQuery& sub_query, + std::string& logtype + ); + + /** + * Generates logtypes and variables for subquery. + * @tparam LogTypeDictionaryReaderType + * @tparam VariableDictionaryReaderType + * @tparam LogTypeDictionaryEntryType + * @param logtype_dict + * @param var_dict + * @param processed_search_string + * @param query_tokens + * @param ignore_case + * @param sub_query + * @return SubQueryMatchabilityResult::SupercedesAllSubQueries + * @return SubQueryMatchabilityResult::WontMatch + * @return SubQueryMatchabilityResult::MayMatch + */ + template < + typename LogTypeDictionaryReaderType, + typename VariableDictionaryReaderType, + typename LogTypeDictionaryEntryType = typename LogTypeDictionaryReaderType::entry_t> + static SubQueryMatchabilityResult generate_logtypes_and_vars_for_subquery( + LogTypeDictionaryReaderType const& logtype_dict, + VariableDictionaryReaderType const& var_dict, + std::string& processed_search_string, + std::vector& query_tokens, + bool ignore_case, + SubQuery& sub_query + ); +}; + +template +std::optional GrepCore::process_raw_query( + LogTypeDictionaryReaderType const& logtype_dict, + VariableDictionaryReaderType const& var_dict, + std::string const& search_string, + epochtime_t search_begin_ts, + epochtime_t search_end_ts, + bool ignore_case, + log_surgeon::lexers::ByteLexer& lexer, + bool use_heuristic +) { + // Split search_string into tokens with wildcards + std::vector query_tokens; + size_t begin_pos = 0; + size_t end_pos = 0; + bool is_var; + std::string search_string_for_sub_queries{search_string}; + if (use_heuristic) { + // Replace '?' wildcards with '*' wildcards since we currently have no support for + // generating sub-queries with '?' wildcards. The final wildcard match on the decompressed + // message uses the original wildcards, so correctness will be maintained. + std::replace( + search_string_for_sub_queries.begin(), + search_string_for_sub_queries.end(), + '?', + '*' + ); + // Clean-up in case any instances of "?*" or "*?" were changed into "**" + search_string_for_sub_queries + = string_utils::clean_up_wildcard_search_string(search_string_for_sub_queries); + while (get_bounds_of_next_potential_var( + search_string_for_sub_queries, + begin_pos, + end_pos, + is_var + )) + { + query_tokens.emplace_back(search_string_for_sub_queries, begin_pos, end_pos, is_var); + } + } else { + while (get_bounds_of_next_potential_var( + search_string_for_sub_queries, + begin_pos, + end_pos, + is_var, + lexer + )) + { + query_tokens.emplace_back(search_string_for_sub_queries, begin_pos, end_pos, is_var); + } + } + + // Get pointers to all ambiguous tokens. Exclude tokens with wildcards in the middle since we + // fall-back to decompression + wildcard matching for those. + std::vector ambiguous_tokens; + for (auto& query_token : query_tokens) { + if (!query_token.has_greedy_wildcard_in_middle() && query_token.is_ambiguous_token()) { + ambiguous_tokens.push_back(&query_token); + } + } + + // Generate a sub-query for each combination of ambiguous tokens + // E.g., if there are two ambiguous tokens each of which could be a logtype or variable, we need + // to create: + // - (token1 as logtype) (token2 as logtype) + // - (token1 as logtype) (token2 as var) + // - (token1 as var) (token2 as logtype) + // - (token1 as var) (token2 as var) + std::vector sub_queries; + std::string logtype; + bool type_of_one_token_changed = true; + while (type_of_one_token_changed) { + SubQuery sub_query; + + // Compute logtypes and variables for query + auto matchability = generate_logtypes_and_vars_for_subquery( + logtype_dict, + var_dict, + search_string_for_sub_queries, + query_tokens, + ignore_case, + sub_query + ); + switch (matchability) { + case SubQueryMatchabilityResult::SupercedesAllSubQueries: + // Since other sub-queries will be superceded by this one, we can stop processing + // now + return Query{search_begin_ts, search_end_ts, ignore_case, search_string, {}}; + case SubQueryMatchabilityResult::MayMatch: + sub_queries.push_back(std::move(sub_query)); + break; + case SubQueryMatchabilityResult::WontMatch: + default: + // Do nothing + break; + } + + // Update combination of ambiguous tokens + type_of_one_token_changed = false; + for (auto* ambiguous_token : ambiguous_tokens) { + if (ambiguous_token->change_to_next_possible_type()) { + type_of_one_token_changed = true; + break; + } + } + } + + if (sub_queries.empty()) { + return std::nullopt; + } + + return Query{ + search_begin_ts, + search_end_ts, + ignore_case, + search_string, + std::move(sub_queries) + }; +} + +template +bool GrepCore::process_var_token( + QueryToken const& query_token, + VariableDictionaryReaderType const& var_dict, + bool ignore_case, + SubQuery& sub_query, + std::string& logtype +) { + // Even though we may have a precise variable, we still fallback to decompressing to ensure that + // it is in the right place in the message + sub_query.mark_wildcard_match_required(); + + // Create QueryVar corresponding to token + if (!query_token.contains_wildcards()) { + if (EncodedVariableInterpreter::encode_and_search_dictionary( + query_token.get_value(), + var_dict, + ignore_case, + logtype, + sub_query + ) + == false) + { + // Variable doesn't exist in dictionary + return false; + } + } else { + if (query_token.has_prefix_greedy_wildcard()) { + logtype += '*'; + } + + if (query_token.is_float_var()) { + EncodedVariableInterpreter::add_float_var(logtype); + } else if (query_token.is_int_var()) { + EncodedVariableInterpreter::add_int_var(logtype); + } else { + EncodedVariableInterpreter::add_dict_var(logtype); + + if (query_token.cannot_convert_to_non_dict_var()) { + // Must be a dictionary variable, so search variable dictionary + if (!EncodedVariableInterpreter::wildcard_search_dictionary_and_get_encoded_matches( + query_token.get_value(), + var_dict, + ignore_case, + sub_query + )) + { + // Variable doesn't exist in dictionary + return false; + } + } + } + + if (query_token.has_suffix_greedy_wildcard()) { + logtype += '*'; + } + } + + return true; +} + +template < + typename LogTypeDictionaryReaderType, + typename VariableDictionaryReaderType, + typename LogTypeDictionaryEntryType> +GrepCore::SubQueryMatchabilityResult GrepCore::generate_logtypes_and_vars_for_subquery( + LogTypeDictionaryReaderType const& logtype_dict, + VariableDictionaryReaderType const& var_dict, + std::string& processed_search_string, + std::vector& query_tokens, + bool ignore_case, + SubQuery& sub_query +) { + size_t last_token_end_pos = 0; + std::string logtype; + auto escape_handler = + [](std::string_view constant, size_t char_to_escape_pos, std::string& logtype) -> void { + auto const escape_char{enum_to_underlying_type(ir::VariablePlaceholder::Escape)}; + auto const next_char_pos{char_to_escape_pos + 1}; + // NOTE: We don't want to add additional escapes for wildcards that have been escaped. E.g., + // the query "\\*" should remain unchanged. + if (next_char_pos < constant.length() + && false == string_utils::is_wildcard(constant[next_char_pos])) + { + logtype += escape_char; + } else if (ir::is_variable_placeholder(constant[char_to_escape_pos])) { + logtype += escape_char; + logtype += escape_char; + } + }; + for (auto const& query_token : query_tokens) { + // Append from end of last token to beginning of this token, to logtype + ir::append_constant_to_logtype( + static_cast(processed_search_string) + .substr(last_token_end_pos, + query_token.get_begin_pos() - last_token_end_pos), + escape_handler, + logtype + ); + last_token_end_pos = query_token.get_end_pos(); + + if (query_token.is_wildcard()) { + logtype += '*'; + } else if (query_token.has_greedy_wildcard_in_middle()) { + // Fallback to decompression + wildcard matching for now to avoid handling queries where + // the pieces of the token on either side of each wildcard need to be processed as + // ambiguous tokens + sub_query.mark_wildcard_match_required(); + if (!query_token.is_var()) { + logtype += '*'; + } else { + logtype += '*'; + EncodedVariableInterpreter::add_dict_var(logtype); + logtype += '*'; + } + } else { + if (!query_token.is_var()) { + ir::append_constant_to_logtype(query_token.get_value(), escape_handler, logtype); + } else if (!process_var_token(query_token, var_dict, ignore_case, sub_query, logtype)) { + return SubQueryMatchabilityResult::WontMatch; + } + } + } + + if (last_token_end_pos < processed_search_string.length()) { + // Append from end of last token to end + ir::append_constant_to_logtype( + static_cast(processed_search_string) + .substr(last_token_end_pos, std::string::npos), + escape_handler, + logtype + ); + last_token_end_pos = processed_search_string.length(); + } + + if ("*" == logtype) { + // Logtype will match all messages + return SubQueryMatchabilityResult::SupercedesAllSubQueries; + } + + // Find matching logtypes + std::unordered_set possible_logtype_entries; + logtype_dict + .get_entries_matching_wildcard_string(logtype, ignore_case, possible_logtype_entries); + if (possible_logtype_entries.empty()) { + return SubQueryMatchabilityResult::WontMatch; + } + + std::unordered_set possible_logtype_ids; + for (auto const* entry : possible_logtype_entries) { + possible_logtype_ids.emplace(entry->get_id()); + } + sub_query.set_possible_logtypes(possible_logtype_ids); + + return SubQueryMatchabilityResult::MayMatch; +} +} // namespace clp + +#endif // CLP_GREPCORE_HPP diff --git a/components/core/src/clp/Query.cpp b/components/core/src/clp/Query.cpp index 45317bfdb8..cac201e615 100644 --- a/components/core/src/clp/Query.cpp +++ b/components/core/src/clp/Query.cpp @@ -1,5 +1,12 @@ #include "Query.hpp" +#include +#include +#include +#include + +#include "Defs.h" + using std::set; using std::string; using std::unordered_set; @@ -30,33 +37,29 @@ QueryVar::QueryVar(encoded_variable_t precise_non_dict_var) { m_precise_var = precise_non_dict_var; m_is_precise_var = true; m_is_dict_var = false; - m_var_dict_entry = nullptr; } -QueryVar::QueryVar( - encoded_variable_t precise_dict_var, - VariableDictionaryEntry const* var_dict_entry -) { +QueryVar::QueryVar(encoded_variable_t precise_dict_var, variable_dictionary_id_t var_dict_id) { m_precise_var = precise_dict_var; m_is_precise_var = true; m_is_dict_var = true; - m_var_dict_entry = var_dict_entry; + m_var_dict_id = var_dict_id; } QueryVar::QueryVar( unordered_set const& possible_dict_vars, - unordered_set const& possible_var_dict_entries + unordered_set const& possible_var_dict_ids ) { m_is_dict_var = true; if (possible_dict_vars.size() == 1) { // A single possible variable is the same as a precise variable m_precise_var = *possible_dict_vars.cbegin(); m_is_precise_var = true; - m_var_dict_entry = *possible_var_dict_entries.cbegin(); + m_var_dict_id = *possible_var_dict_ids.cbegin(); } else { m_possible_dict_vars = possible_dict_vars; m_is_precise_var = false; - m_possible_var_dict_entries = possible_var_dict_entries; + m_possible_var_dict_ids = possible_var_dict_ids; } } @@ -65,7 +68,11 @@ bool QueryVar::matches(encoded_variable_t var) const { || (!m_is_precise_var && m_possible_dict_vars.count(var) > 0); } -void QueryVar::remove_segments_that_dont_contain_dict_var(set& segment_ids) const { +void QueryVar::remove_segments_that_dont_contain_dict_var( + set& segment_ids, + std::function const&(variable_dictionary_id_t)> const& + get_segments_containing_var_dict_id +) const { if (false == m_is_dict_var) { // Not a dictionary variable, so do nothing return; @@ -73,12 +80,12 @@ void QueryVar::remove_segments_that_dont_contain_dict_var(set& seg if (m_is_precise_var) { auto& ids_of_segments_containing_query_var - = m_var_dict_entry->get_ids_of_segments_containing_entry(); + = get_segments_containing_var_dict_id(m_var_dict_id); inplace_set_intersection(ids_of_segments_containing_query_var, segment_ids); } else { set ids_of_segments_containing_query_var; - for (auto entry : m_possible_var_dict_entries) { - auto& ids_of_segments_containing_var = entry->get_ids_of_segments_containing_entry(); + for (auto var_dict_id : m_possible_var_dict_ids) { + auto& ids_of_segments_containing_var = get_segments_containing_var_dict_id(var_dict_id); ids_of_segments_containing_query_var.insert( ids_of_segments_containing_var.cbegin(), ids_of_segments_containing_var.cend() @@ -92,39 +99,37 @@ void SubQuery::add_non_dict_var(encoded_variable_t precise_non_dict_var) { m_vars.emplace_back(precise_non_dict_var); } -void SubQuery::add_dict_var( - encoded_variable_t precise_dict_var, - VariableDictionaryEntry const* var_dict_entry -) { - m_vars.emplace_back(precise_dict_var, var_dict_entry); +void +SubQuery::add_dict_var(encoded_variable_t precise_dict_var, variable_dictionary_id_t var_dict_id) { + m_vars.emplace_back(precise_dict_var, var_dict_id); } void SubQuery::add_imprecise_dict_var( unordered_set const& possible_dict_vars, - unordered_set const& possible_var_dict_entries + unordered_set const& possible_var_dict_ids ) { - m_vars.emplace_back(possible_dict_vars, possible_var_dict_entries); + m_vars.emplace_back(possible_dict_vars, possible_var_dict_ids); } -void SubQuery::set_possible_logtypes( - unordered_set const& logtype_entries -) { - m_possible_logtype_ids.clear(); - for (auto entry : logtype_entries) { - m_possible_logtype_ids.insert(entry->get_id()); - } - m_possible_logtype_entries = logtype_entries; +void SubQuery::set_possible_logtypes(unordered_set const& logtype_ids) { + m_possible_logtypes = logtype_ids; } void SubQuery::mark_wildcard_match_required() { m_wildcard_match_required = true; } -void SubQuery::calculate_ids_of_matching_segments() { +void SubQuery::calculate_ids_of_matching_segments( + std::function const&(logtype_dictionary_id_t)> const& + get_segments_containing_logtype_dict_id, + std::function const&(variable_dictionary_id_t)> const& + get_segments_containing_var_dict_id +) { // Get IDs of segments containing logtypes m_ids_of_matching_segments.clear(); - for (auto entry : m_possible_logtype_entries) { - auto& ids_of_segments_containing_logtype = entry->get_ids_of_segments_containing_entry(); + for (auto logtype_id : m_possible_logtypes) { + auto const& ids_of_segments_containing_logtype + = get_segments_containing_logtype_dict_id(logtype_id); m_ids_of_matching_segments.insert( ids_of_segments_containing_logtype.cbegin(), ids_of_segments_containing_logtype.cend() @@ -133,43 +138,21 @@ void SubQuery::calculate_ids_of_matching_segments() { // Intersect with IDs of segments containing variables for (auto& query_var : m_vars) { - query_var.remove_segments_that_dont_contain_dict_var(m_ids_of_matching_segments); + query_var.remove_segments_that_dont_contain_dict_var( + m_ids_of_matching_segments, + get_segments_containing_var_dict_id + ); } } void SubQuery::clear() { m_vars.clear(); - m_possible_logtype_ids.clear(); + m_possible_logtypes.clear(); m_wildcard_match_required = false; } bool SubQuery::matches_logtype(logtype_dictionary_id_t const logtype) const { - return m_possible_logtype_ids.count(logtype) > 0; -} - -bool SubQuery::matches_vars(std::vector const& vars) const { - if (vars.size() < m_vars.size()) { - // Not enough variables to satisfy query - return false; - } - - // Try to find m_vars in vars, in order, but not necessarily contiguously - size_t possible_vars_ix = 0; - size_t const num_possible_vars = m_vars.size(); - size_t vars_ix = 0; - size_t const num_vars = vars.size(); - while (possible_vars_ix < num_possible_vars && vars_ix < num_vars) { - QueryVar const& possible_var = m_vars[possible_vars_ix]; - - if (possible_var.matches(vars[vars_ix])) { - // Matched - ++possible_vars_ix; - ++vars_ix; - } else { - ++vars_ix; - } - } - return (num_possible_vars == possible_vars_ix); + return m_possible_logtypes.count(logtype) > 0; } Query::Query( @@ -202,4 +185,18 @@ void Query::make_sub_queries_relevant_to_segment(segment_id_t segment_id) { } m_prev_segment_id = segment_id; } + +void Query::calculate_ids_of_matching_segments( + std::function const&(logtype_dictionary_id_t)> const& + get_segments_containing_logtype_dict_id, + std::function const&(variable_dictionary_id_t)> const& + get_segments_containing_var_dict_id +) { + for (auto& sub_query : m_sub_queries) { + sub_query.calculate_ids_of_matching_segments( + get_segments_containing_logtype_dict_id, + get_segments_containing_var_dict_id + ); + } +} } // namespace clp diff --git a/components/core/src/clp/Query.hpp b/components/core/src/clp/Query.hpp index 20e780e1ec..f28989096b 100644 --- a/components/core/src/clp/Query.hpp +++ b/components/core/src/clp/Query.hpp @@ -1,14 +1,13 @@ #ifndef CLP_QUERY_HPP #define CLP_QUERY_HPP +#include #include #include #include #include #include "Defs.h" -#include "LogTypeDictionaryEntry.hpp" -#include "VariableDictionaryEntry.hpp" namespace clp { /** @@ -19,10 +18,10 @@ class QueryVar { public: // Constructors explicit QueryVar(encoded_variable_t precise_non_dict_var); - QueryVar(encoded_variable_t precise_dict_var, VariableDictionaryEntry const* var_dict_entry); + QueryVar(encoded_variable_t precise_dict_var, variable_dictionary_id_t var_dict_id); QueryVar( std::unordered_set const& possible_dict_vars, - std::unordered_set const& possible_var_dict_entries + std::unordered_set const& possible_var_dict_ids ); // Methods @@ -36,32 +35,36 @@ class QueryVar { /** * Removes segments from the given set that don't contain the given variable * @param segment_ids + * @param get_segments_containing_var_dict_id */ - void remove_segments_that_dont_contain_dict_var(std::set& segment_ids) const; + void remove_segments_that_dont_contain_dict_var( + std::set& segment_ids, + std::function const&(variable_dictionary_id_t)> const& + get_segments_containing_var_dict_id + ) const; bool is_precise_var() const { return m_is_precise_var; } bool is_dict_var() const { return m_is_dict_var; } - VariableDictionaryEntry const* get_var_dict_entry() const { return m_var_dict_entry; } + variable_dictionary_id_t get_var_dict_id() const { return m_var_dict_id; } - std::unordered_set const& - get_possible_var_dict_entries() const { - return m_possible_var_dict_entries; + std::unordered_set const& get_possible_var_dict_ids() const { + return m_possible_var_dict_ids; } private: // Variables - bool m_is_precise_var; - bool m_is_dict_var; + bool m_is_precise_var{}; + bool m_is_dict_var{}; - encoded_variable_t m_precise_var; + encoded_variable_t m_precise_var{}; // Only used if the precise variable is a dictionary variable - VariableDictionaryEntry const* m_var_dict_entry; + variable_dictionary_id_t m_var_dict_id{}; // Only used if the variable is an imprecise dictionary variable std::unordered_set m_possible_dict_vars; - std::unordered_set m_possible_var_dict_entries; + std::unordered_set m_possible_var_dict_ids; }; /** @@ -80,45 +83,47 @@ class SubQuery { /** * Adds a precise dictionary variable to the subquery * @param precise_dict_var - * @param var_dict_entry + * @param var_dict_id */ - void add_dict_var( - encoded_variable_t precise_dict_var, - VariableDictionaryEntry const* var_dict_entry - ); + void add_dict_var(encoded_variable_t precise_dict_var, variable_dictionary_id_t var_dict_id); /** * Adds an imprecise dictionary variable (i.e., a set of possible precise dictionary variables) * to the subquery * @param possible_dict_vars - * @param possible_var_dict_entries + * @param possible_var_dict_ids */ void add_imprecise_dict_var( std::unordered_set const& possible_dict_vars, - std::unordered_set const& possible_var_dict_entries + std::unordered_set const& possible_var_dict_ids ); /** * Add a set of possible logtypes to the subquery - * @param logtype_entries + * @param logtype_ids */ - void set_possible_logtypes( - std::unordered_set const& logtype_entries - ); + void set_possible_logtypes(std::unordered_set const& logtype_ids); void mark_wildcard_match_required(); /** * Calculates the segment IDs that should contain a match for the subquery's current logtypes - * and QueryVars + * and QueryVars. + * @param get_segments_containing_logtype_dict_id + * @param get_segments_containing_var_dict_id */ - void calculate_ids_of_matching_segments(); + void calculate_ids_of_matching_segments( + std::function const&(logtype_dictionary_id_t)> const& + get_segments_containing_logtype_dict_id, + std::function const&(variable_dictionary_id_t)> const& + get_segments_containing_var_dict_id + ); void clear(); bool wildcard_match_required() const { return m_wildcard_match_required; } - size_t get_num_possible_logtypes() const { return m_possible_logtype_ids.size(); } + size_t get_num_possible_logtypes() const { return m_possible_logtypes.size(); } - std::unordered_set const& get_possible_logtype_entries() const { - return m_possible_logtype_entries; + std::unordered_set const& get_possible_logtypes() const { + return m_possible_logtypes; } size_t get_num_possible_vars() const { return m_vars.size(); } @@ -138,15 +143,16 @@ class SubQuery { /** * Whether the given variables contain the subquery's variables in order (but not necessarily * contiguously) + * @tparam EncodedVariableContainerType A random access list of `clp::encoded_variable_t`. * @param vars * @return true if matched, false otherwise */ - bool matches_vars(std::vector const& vars) const; + template + bool matches_vars(EncodedVariableContainerType const& vars) const; private: // Variables - std::unordered_set m_possible_logtype_entries; - std::unordered_set m_possible_logtype_ids; + std::unordered_set m_possible_logtypes; std::set m_ids_of_matching_segments; std::vector m_vars; bool m_wildcard_match_required; @@ -204,6 +210,19 @@ class Query { return m_relevant_sub_queries; } + /** + * Calculates the segment IDs that should contain a match for each subquery's logtypes and + * QueryVars. + * @param get_segments_containing_logtype_dict_id + * @param get_segments_containing_var_dict_id + */ + void calculate_ids_of_matching_segments( + std::function const&(logtype_dictionary_id_t)> const& + get_segments_containing_logtype_dict_id, + std::function const&(variable_dictionary_id_t)> const& + get_segments_containing_var_dict_id + ); + private: // Variables // Start of search time range (inclusive) @@ -217,6 +236,32 @@ class Query { std::vector m_relevant_sub_queries; segment_id_t m_prev_segment_id{cInvalidSegmentId}; }; + +template +bool SubQuery::matches_vars(EncodedVariableContainerType const& vars) const { + if (vars.size() < m_vars.size()) { + // Not enough variables to satisfy query + return false; + } + + // Try to find m_vars in vars, in order, but not necessarily contiguously + size_t possible_vars_ix = 0; + size_t const num_possible_vars = m_vars.size(); + size_t vars_ix = 0; + size_t const num_vars = vars.size(); + while (possible_vars_ix < num_possible_vars && vars_ix < num_vars) { + QueryVar const& possible_var = m_vars[possible_vars_ix]; + + if (possible_var.matches(vars[vars_ix])) { + // Matched + ++possible_vars_ix; + ++vars_ix; + } else { + ++vars_ix; + } + } + return (num_possible_vars == possible_vars_ix); +} } // namespace clp #endif // CLP_QUERY_HPP diff --git a/components/core/src/clp/QueryToken.cpp b/components/core/src/clp/QueryToken.cpp new file mode 100644 index 0000000000..171285aee9 --- /dev/null +++ b/components/core/src/clp/QueryToken.cpp @@ -0,0 +1,172 @@ +#include "QueryToken.hpp" + +#include + +#include "Defs.h" +#include "EncodedVariableInterpreter.hpp" + +using std::string; + +namespace clp { +QueryToken::QueryToken( + string const& query_string, + size_t const begin_pos, + size_t const end_pos, + bool const is_var +) + : m_current_possible_type_ix(0) { + m_begin_pos = begin_pos; + m_end_pos = end_pos; + m_value.assign(query_string, m_begin_pos, m_end_pos - m_begin_pos); + + // Set wildcard booleans and determine type + if ("*" == m_value) { + m_has_prefix_greedy_wildcard = true; + m_has_suffix_greedy_wildcard = false; + m_has_greedy_wildcard_in_middle = false; + m_contains_wildcards = true; + m_type = Type::Wildcard; + } else { + m_has_prefix_greedy_wildcard = ('*' == m_value[0]); + m_has_suffix_greedy_wildcard = ('*' == m_value[m_value.length() - 1]); + + m_has_greedy_wildcard_in_middle = false; + for (size_t i = 1; i < m_value.length() - 1; ++i) { + if ('*' == m_value[i]) { + m_has_greedy_wildcard_in_middle = true; + break; + } + } + + m_contains_wildcards + = (m_has_prefix_greedy_wildcard || m_has_suffix_greedy_wildcard + || m_has_greedy_wildcard_in_middle); + + if (!is_var) { + if (!m_contains_wildcards) { + m_type = Type::Logtype; + } else { + m_type = Type::Ambiguous; + m_possible_types.push_back(Type::Logtype); + m_possible_types.push_back(Type::IntVar); + m_possible_types.push_back(Type::FloatVar); + m_possible_types.push_back(Type::DictionaryVar); + } + } else { + string value_without_wildcards = m_value; + if (m_has_prefix_greedy_wildcard) { + value_without_wildcards = value_without_wildcards.substr(1); + } + if (m_has_suffix_greedy_wildcard) { + value_without_wildcards.resize(value_without_wildcards.length() - 1); + } + + encoded_variable_t encoded_var; + bool converts_to_non_dict_var = false; + if (EncodedVariableInterpreter::convert_string_to_representable_integer_var( + value_without_wildcards, + encoded_var + ) + || EncodedVariableInterpreter::convert_string_to_representable_float_var( + value_without_wildcards, + encoded_var + )) + { + converts_to_non_dict_var = true; + } + + if (!converts_to_non_dict_var) { + // Dictionary variable + m_type = Type::DictionaryVar; + m_cannot_convert_to_non_dict_var = true; + } else { + m_type = Type::Ambiguous; + m_possible_types.push_back(Type::IntVar); + m_possible_types.push_back(Type::FloatVar); + m_possible_types.push_back(Type::DictionaryVar); + m_cannot_convert_to_non_dict_var = false; + } + } + } +} + +bool QueryToken::cannot_convert_to_non_dict_var() const { + return m_cannot_convert_to_non_dict_var; +} + +bool QueryToken::contains_wildcards() const { + return m_contains_wildcards; +} + +bool QueryToken::has_greedy_wildcard_in_middle() const { + return m_has_greedy_wildcard_in_middle; +} + +bool QueryToken::has_prefix_greedy_wildcard() const { + return m_has_prefix_greedy_wildcard; +} + +bool QueryToken::has_suffix_greedy_wildcard() const { + return m_has_suffix_greedy_wildcard; +} + +bool QueryToken::is_ambiguous_token() const { + return Type::Ambiguous == m_type; +} + +bool QueryToken::is_float_var() const { + Type type; + if (Type::Ambiguous == m_type) { + type = m_possible_types[m_current_possible_type_ix]; + } else { + type = m_type; + } + return Type::FloatVar == type; +} + +bool QueryToken::is_int_var() const { + Type type; + if (Type::Ambiguous == m_type) { + type = m_possible_types[m_current_possible_type_ix]; + } else { + type = m_type; + } + return Type::IntVar == type; +} + +bool QueryToken::is_var() const { + Type type; + if (Type::Ambiguous == m_type) { + type = m_possible_types[m_current_possible_type_ix]; + } else { + type = m_type; + } + return (Type::IntVar == type || Type::FloatVar == type || Type::DictionaryVar == type); +} + +bool QueryToken::is_wildcard() const { + return Type::Wildcard == m_type; +} + +size_t QueryToken::get_begin_pos() const { + return m_begin_pos; +} + +size_t QueryToken::get_end_pos() const { + return m_end_pos; +} + +string const& QueryToken::get_value() const { + return m_value; +} + +bool QueryToken::change_to_next_possible_type() { + if (m_current_possible_type_ix < m_possible_types.size() - 1) { + ++m_current_possible_type_ix; + return true; + } else { + m_current_possible_type_ix = 0; + return false; + } +} +} // namespace clp diff --git a/components/core/src/clp/QueryToken.hpp b/components/core/src/clp/QueryToken.hpp new file mode 100644 index 0000000000..0d339b52f2 --- /dev/null +++ b/components/core/src/clp/QueryToken.hpp @@ -0,0 +1,66 @@ +#ifndef CLP_QUERYTOKEN_HPP +#define CLP_QUERYTOKEN_HPP + +#include +#include +#include + +namespace clp { +// Class representing a token in a query. It is used to interpret a token in a search string. +class QueryToken { +public: + // Constructors + QueryToken(std::string const& query_string, size_t begin_pos, size_t end_pos, bool is_var); + + // Methods + bool cannot_convert_to_non_dict_var() const; + bool contains_wildcards() const; + bool has_greedy_wildcard_in_middle() const; + bool has_prefix_greedy_wildcard() const; + bool has_suffix_greedy_wildcard() const; + bool is_ambiguous_token() const; + bool is_float_var() const; + bool is_int_var() const; + bool is_var() const; + bool is_wildcard() const; + + size_t get_begin_pos() const; + size_t get_end_pos() const; + std::string const& get_value() const; + + bool change_to_next_possible_type(); + +private: + // Types + // Type for the purpose of generating different subqueries. E.g., if a token is of type + // DictOrIntVar, it would generate a different subquery than if it was of type Logtype. + enum class Type { + Wildcard, + // Ambiguous indicates the token can be more than one of the types listed below + Ambiguous, + Logtype, + DictionaryVar, + FloatVar, + IntVar + }; + + // Variables + bool m_cannot_convert_to_non_dict_var{}; + bool m_contains_wildcards{}; + bool m_has_greedy_wildcard_in_middle{}; + bool m_has_prefix_greedy_wildcard{}; + bool m_has_suffix_greedy_wildcard{}; + + size_t m_begin_pos{}; + size_t m_end_pos{}; + std::string m_value; + + // Type if variable has unambiguous type + Type m_type{Type::Wildcard}; + // Types if variable type is ambiguous + std::vector m_possible_types; + // Index of the current possible type selected for generating a subquery + size_t m_current_possible_type_ix{}; +}; +} // namespace clp +#endif // CLP_QUERYTOKEN_HPP diff --git a/components/core/src/clp/clg/CMakeLists.txt b/components/core/src/clp/clg/CMakeLists.txt index 816173bdb0..d4443536d7 100644 --- a/components/core/src/clp/clg/CMakeLists.txt +++ b/components/core/src/clp/clg/CMakeLists.txt @@ -33,6 +33,8 @@ set( ../GlobalSQLiteMetadataDB.hpp ../Grep.cpp ../Grep.hpp + ../GrepCore.cpp + ../GrepCore.hpp ../ir/EncodedTextAst.cpp ../ir/EncodedTextAst.hpp ../ir/LogEvent.hpp @@ -59,6 +61,8 @@ set( ../Profiler.hpp ../Query.cpp ../Query.hpp + ../QueryToken.cpp + ../QueryToken.hpp ../ReaderInterface.cpp ../ReaderInterface.hpp ../ReadOnlyMemoryMappedFile.cpp diff --git a/components/core/src/clp/clg/clg.cpp b/components/core/src/clp/clg/clg.cpp index 5faa8e0cc8..04d5e0a901 100644 --- a/components/core/src/clp/clg/clg.cpp +++ b/components/core/src/clp/clg/clg.cpp @@ -2,14 +2,17 @@ #include #include +#include #include #include +#include #include "../Defs.h" #include "../GlobalMySQLMetadataDB.hpp" #include "../GlobalSQLiteMetadataDB.hpp" #include "../Grep.hpp" +#include "../GrepCore.hpp" #include "../Profiler.hpp" #include "../spdlog_with_specializations.hpp" #include "../streaming_archive/Constants.hpp" @@ -25,7 +28,9 @@ using clp::FileReader; using clp::GlobalMetadataDB; using clp::GlobalMetadataDBConfig; using clp::Grep; +using clp::GrepCore; using clp::load_lexer_from_file; +using clp::logtype_dictionary_id_t; using clp::Profiler; using clp::Query; using clp::segment_id_t; @@ -33,7 +38,9 @@ using clp::streaming_archive::MetadataDB; using clp::streaming_archive::reader::Archive; using clp::streaming_archive::reader::File; using clp::streaming_archive::reader::Message; +using clp::string_utils::clean_up_wildcard_search_string; using clp::TraceableException; +using clp::variable_dictionary_id_t; using std::cerr; using std::cout; using std::endl; @@ -205,8 +212,11 @@ static bool search( std::set ids_of_segments_to_search; bool is_superseding_query = false; for (auto const& search_string : search_strings) { - auto query_processing_result = Grep::process_raw_query( - archive, + auto const& logtype_dict{archive.get_logtype_dictionary()}; + auto const& var_dict{archive.get_var_dictionary()}; + auto query_processing_result = GrepCore::process_raw_query( + logtype_dict, + var_dict, search_string, search_begin_ts, search_end_ts, @@ -229,6 +239,24 @@ static bool search( break; } + // Calculate the IDs of the segments that may contain results for each sub-query. + auto get_segments_containing_logtype_dict_id + = [&logtype_dict]( + logtype_dictionary_id_t logtype_id + ) -> std::set const& { + return logtype_dict.get_entry(logtype_id) + .get_ids_of_segments_containing_entry(); + }; + auto get_segments_containing_var_dict_id = [&var_dict]( + variable_dictionary_id_t var_id + ) -> std::set const& { + return var_dict.get_entry(var_id).get_ids_of_segments_containing_entry(); + }; + query.calculate_ids_of_matching_segments( + get_segments_containing_logtype_dict_id, + get_segments_containing_var_dict_id + ); + queries.push_back(query); // Add query's matching segments to segments to search @@ -474,16 +502,20 @@ int main(int argc, char const* argv[]) { Profiler::start_continuous_measurement(); + auto add_implicit_wildcards = [](string const& search_string) -> string { + return clean_up_wildcard_search_string('*' + search_string + '*'); + }; + // Create vector of search strings vector search_strings; if (command_line_args.get_search_strings_file_path().empty()) { - search_strings.push_back(command_line_args.get_search_string()); + search_strings.emplace_back(add_implicit_wildcards(command_line_args.get_search_string())); } else { FileReader file_reader{command_line_args.get_search_strings_file_path()}; string line; while (file_reader.read_to_delimiter('\n', false, false, line)) { if (!line.empty()) { - search_strings.push_back(line); + search_strings.emplace_back(add_implicit_wildcards(line)); } } } diff --git a/components/core/src/clp/clo/CMakeLists.txt b/components/core/src/clp/clo/CMakeLists.txt index 94d03c79e0..fef53eb8de 100644 --- a/components/core/src/clp/clo/CMakeLists.txt +++ b/components/core/src/clp/clo/CMakeLists.txt @@ -34,6 +34,8 @@ set( ../FileWriter.hpp ../Grep.cpp ../Grep.hpp + ../GrepCore.cpp + ../GrepCore.hpp ../ir/EncodedTextAst.cpp ../ir/EncodedTextAst.hpp ../ir/LogEvent.hpp @@ -59,6 +61,8 @@ set( ../Profiler.hpp ../Query.cpp ../Query.hpp + ../QueryToken.cpp + ../QueryToken.hpp ../ReaderInterface.cpp ../ReaderInterface.hpp ../ReadOnlyMemoryMappedFile.cpp diff --git a/components/core/src/clp/clo/clo.cpp b/components/core/src/clp/clo/clo.cpp index b94ffb9555..94b9e793b6 100644 --- a/components/core/src/clp/clo/clo.cpp +++ b/components/core/src/clp/clo/clo.cpp @@ -1,16 +1,19 @@ #include #include #include +#include #include #include #include #include +#include #include "../../reducer/network_utils.hpp" #include "../clp/FileDecompressor.hpp" #include "../Defs.h" #include "../Grep.hpp" +#include "../GrepCore.hpp" #include "../ir/constants.hpp" #include "../Profiler.hpp" #include "../spdlog_with_specializations.hpp" @@ -35,14 +38,19 @@ using clp::ErrorCode_errno; using clp::ErrorCode_FileExists; using clp::ErrorCode_Success; using clp::Grep; +using clp::GrepCore; using clp::ir::cIrFileExtension; using clp::load_lexer_from_file; +using clp::logtype_dictionary_id_t; using clp::Query; +using clp::segment_id_t; using clp::streaming_archive::MetadataDB; using clp::streaming_archive::reader::Archive; using clp::streaming_archive::reader::File; using clp::streaming_archive::reader::Message; +using clp::string_utils::clean_up_wildcard_search_string; using clp::TraceableException; +using clp::variable_dictionary_id_t; using std::cerr; using std::cout; using std::endl; @@ -489,10 +497,15 @@ static bool search_archive( auto search_begin_ts = command_line_args.get_search_begin_ts(); auto search_end_ts = command_line_args.get_search_end_ts(); - - auto query_processing_result = Grep::process_raw_query( - archive_reader, - command_line_args.get_search_string(), + auto const& logtype_dict{archive_reader.get_logtype_dictionary()}; + auto const& var_dict{archive_reader.get_var_dictionary()}; + + std::string wildcard_search_string + = clean_up_wildcard_search_string('*' + command_line_args.get_search_string() + '*'); + auto query_processing_result = GrepCore::process_raw_query( + logtype_dict, + var_dict, + wildcard_search_string, search_begin_ts, search_end_ts, command_line_args.ignore_case(), @@ -504,6 +517,20 @@ static bool search_archive( } auto& query = query_processing_result.value(); + // Calculate the IDs of the segments that may contain results for each sub-query. + auto get_segments_containing_logtype_dict_id + = [&logtype_dict](logtype_dictionary_id_t logtype_id) -> std::set const& { + return logtype_dict.get_entry(logtype_id).get_ids_of_segments_containing_entry(); + }; + auto get_segments_containing_var_dict_id + = [&var_dict](variable_dictionary_id_t var_id) -> std::set const& { + return var_dict.get_entry(var_id).get_ids_of_segments_containing_entry(); + }; + query.calculate_ids_of_matching_segments( + get_segments_containing_logtype_dict_id, + get_segments_containing_var_dict_id + ); + // Get all segments potentially containing query results std::set ids_of_segments_to_search; for (auto& sub_query : query.get_sub_queries()) { diff --git a/components/core/tests/test-Grep.cpp b/components/core/tests/test-GrepCore.cpp similarity index 55% rename from components/core/tests/test-Grep.cpp rename to components/core/tests/test-GrepCore.cpp index e4a256c5d5..c103d66b9d 100644 --- a/components/core/tests/test-Grep.cpp +++ b/components/core/tests/test-GrepCore.cpp @@ -4,9 +4,10 @@ #include #include -#include "../src/clp/Grep.hpp" +#include "../src/clp/GrepCore.hpp" +#include "../src/clp/Utils.hpp" -using clp::Grep; +using clp::GrepCore; using clp::load_lexer_from_file; using log_surgeon::DelimiterStringAST; using log_surgeon::lexers::ByteLexer; @@ -29,21 +30,21 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var str = ""; begin_pos = string::npos; end_pos = string::npos; - REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) + REQUIRE(GrepCore::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == false); // Empty string str = ""; begin_pos = 0; end_pos = 0; - REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) + REQUIRE(GrepCore::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == false); // No tokens str = "="; begin_pos = 0; end_pos = 0; - REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) + REQUIRE(GrepCore::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == false); // No wildcards @@ -51,39 +52,46 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos = 0; end_pos = 0; - REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); + REQUIRE(GrepCore::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) + == true); REQUIRE("95" == str.substr(begin_pos, end_pos - begin_pos)); REQUIRE(true == is_var); - REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); + REQUIRE(GrepCore::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) + == true); REQUIRE("ad" == str.substr(begin_pos, end_pos - begin_pos)); REQUIRE(true == is_var); - REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); + REQUIRE(GrepCore::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) + == true); REQUIRE("ff" == str.substr(begin_pos, end_pos - begin_pos)); REQUIRE(true == is_var); - REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); + REQUIRE(GrepCore::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) + == true); REQUIRE("95" == str.substr(begin_pos, end_pos - begin_pos)); REQUIRE(true == is_var); - REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); + REQUIRE(GrepCore::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) + == true); REQUIRE("24" == str.substr(begin_pos, end_pos - begin_pos)); REQUIRE(true == is_var); - REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); + REQUIRE(GrepCore::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) + == true); REQUIRE("0d" == str.substr(begin_pos, end_pos - begin_pos)); REQUIRE(true == is_var); - REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); + REQUIRE(GrepCore::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) + == true); REQUIRE("ff" == str.substr(begin_pos, end_pos - begin_pos)); REQUIRE(true == is_var); - REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var) == true); + REQUIRE(GrepCore::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var) == true); REQUIRE("-abc-" == str.substr(begin_pos, end_pos - begin_pos)); REQUIRE(true == is_var); - REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) + REQUIRE(GrepCore::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == false); REQUIRE(str.length() == begin_pos); @@ -92,28 +100,33 @@ TEST_CASE("get_bounds_of_next_potential_var", "[get_bounds_of_next_potential_var begin_pos = 0; end_pos = 0; - REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); + REQUIRE(GrepCore::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) + == true); REQUIRE(str.substr(begin_pos, end_pos - begin_pos) == "1\\*x"); REQUIRE(is_var == true); // REQUIRE(is_var == true); - REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); + REQUIRE(GrepCore::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) + == true); REQUIRE(str.substr(begin_pos, end_pos - begin_pos) == "abc*123"); REQUIRE(is_var == false); // REQUIRE(is_var == true); - REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); + REQUIRE(GrepCore::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) + == true); REQUIRE(str.substr(begin_pos, end_pos - begin_pos) == "1.2"); REQUIRE(is_var == true); - REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); + REQUIRE(GrepCore::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) + == true); REQUIRE(str.substr(begin_pos, end_pos - begin_pos) == "+394/-"); REQUIRE(is_var == true); - REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == true); + REQUIRE(GrepCore::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) + == true); REQUIRE(str.substr(begin_pos, end_pos - begin_pos) == "-*abc-"); REQUIRE(is_var == false); - REQUIRE(Grep::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) + REQUIRE(GrepCore::get_bounds_of_next_potential_var(str, begin_pos, end_pos, is_var, lexer) == false); } diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index add1e1eaca..7c704dc96d 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -282,6 +282,8 @@ tasks: - "clp/GlobalSQLiteMetadataDB.hpp" - "clp/Grep.cpp" - "clp/Grep.hpp" + - "clp/GrepCore.cpp" + - "clp/GrepCore.hpp" - "clp/hash_utils.cpp" - "clp/ir/EncodedTextAst.cpp" - "clp/ir/LogEvent.hpp" @@ -324,6 +326,8 @@ tasks: - "clp/Profiler.hpp" - "clp/Query.cpp" - "clp/Query.hpp" + - "clp/QueryToken.cpp" + - "clp/QueryToken.hpp" - "clp/ReaderInterface.cpp" - "clp/ReaderInterface.hpp" - "clp/spdlog_with_specializations.hpp" @@ -727,7 +731,7 @@ tasks: - "test-encoding_methods.cpp" - "test-error_handling.cpp" - "test-ffi_IrUnitHandlerInterface.cpp" - - "test-Grep.cpp" + - "test-GrepCore.cpp" - "test-hash_utils.cpp" - "test-ir_encoding_methods.cpp" - "test-ir_parsing.cpp"