From f268c82380754d809b4a727d75a15cb10b1cf35d Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 14 Jul 2025 18:41:04 +0000 Subject: [PATCH] Delete clp_s::StringUtils functions that have equivalent implementations in clp::string_utils. --- components/core/cmake/Options/options.cmake | 2 + components/core/src/clp_s/CMakeLists.txt | 1 + .../core/src/clp_s/DictionaryReader.hpp | 8 +- components/core/src/clp_s/Utils.cpp | 307 +----------------- components/core/src/clp_s/Utils.hpp | 143 -------- .../core/src/clp_s/search/CMakeLists.txt | 1 + .../core/src/clp_s/search/QueryRunner.cpp | 13 +- .../core/src/clp_s/search/clp_search/Grep.cpp | 21 +- 8 files changed, 33 insertions(+), 463 deletions(-) diff --git a/components/core/cmake/Options/options.cmake b/components/core/cmake/Options/options.cmake index ad125cfcc4..40b1c610e7 100644 --- a/components/core/cmake/Options/options.cmake +++ b/components/core/cmake/Options/options.cmake @@ -198,6 +198,7 @@ endfunction() function(validate_clp_s_archivereader_dependencies) validate_clp_dependencies_for_target(CLP_BUILD_CLP_S_ARCHIVEREADER + CLP_BUILD_CLP_STRING_UTILS CLP_BUILD_CLP_S_CLP_DEPENDENCIES CLP_BUILD_CLP_S_IO CLP_BUILD_CLP_S_TIMESTAMPPATTERN @@ -300,6 +301,7 @@ endfunction() function(validate_clp_s_search_dependencies) validate_clp_dependencies_for_target(CLP_BUILD_CLP_S_SEARCH + CLP_BUILD_CLP_STRING_UTILS CLP_BUILD_CLP_S_ARCHIVEREADER CLP_BUILD_CLP_S_CLP_DEPENDENCIES CLP_BUILD_CLP_S_SEARCH_AST diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 2e4cae6e81..ef3b34b2fc 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -302,6 +302,7 @@ if(CLP_BUILD_CLP_S_ARCHIVEREADER) clp_s_archive_reader PUBLIC absl::flat_hash_map + clp::string_utils clp_s::io msgpack-cxx nlohmann_json::nlohmann_json diff --git a/components/core/src/clp_s/DictionaryReader.hpp b/components/core/src/clp_s/DictionaryReader.hpp index 7f05e9bd49..2d378d31f6 100644 --- a/components/core/src/clp_s/DictionaryReader.hpp +++ b/components/core/src/clp_s/DictionaryReader.hpp @@ -6,6 +6,7 @@ #include #include +#include #include "ArchiveReaderAdaptor.hpp" #include "DictionaryEntry.hpp" @@ -191,7 +192,12 @@ void DictionaryReader::get_entries_matching_wildcar std::unordered_set& entries ) const { for (auto const& entry : m_entries) { - if (StringUtils::wildcard_match_unsafe(entry.get_value(), wildcard_string, !ignore_case)) { + if (clp::string_utils::wildcard_match_unsafe( + entry.get_value(), + wildcard_string, + !ignore_case + )) + { entries.insert(&entry); } } diff --git a/components/core/src/clp_s/Utils.cpp b/components/core/src/clp_s/Utils.cpp index 2339784436..d278daca63 100644 --- a/components/core/src/clp_s/Utils.cpp +++ b/components/core/src/clp_s/Utils.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include "archive_constants.hpp" @@ -188,9 +189,9 @@ bool StringUtils::get_bounds_of_next_var(string const& msg, size_t& begin_pos, s end_pos = begin_pos; for (; end_pos < msg_length; ++end_pos) { char c = msg[end_pos]; - if (is_decimal_digit(c)) { + if (clp::string_utils::is_decimal_digit(c)) { contains_decimal_digit = true; - } else if (is_alphabet(c)) { + } else if (clp::string_utils::is_alphabet(c)) { contains_alphabet = true; } else if (is_delim(c)) { break; @@ -212,308 +213,6 @@ bool StringUtils::get_bounds_of_next_var(string const& msg, size_t& begin_pos, s return (msg_length != begin_pos); } -size_t StringUtils::find_first_of( - string const& haystack, - char const* needles, - size_t search_start_pos, - size_t& needle_ix -) { - size_t haystack_length = haystack.length(); - size_t needles_length = strlen(needles); - for (size_t i = search_start_pos; i < haystack_length; ++i) { - for (needle_ix = 0; needle_ix < needles_length; ++needle_ix) { - if (haystack[i] == needles[needle_ix]) { - return i; - } - } - } - - return string::npos; -} - -string StringUtils::replace_characters( - char const* characters_to_escape, - char const* replacement_characters, - string const& value, - bool escape -) { - string new_value; - size_t search_start_pos = 0; - while (true) { - size_t replace_char_ix; - size_t char_to_replace_pos - = find_first_of(value, characters_to_escape, search_start_pos, replace_char_ix); - if (string::npos == char_to_replace_pos) { - new_value.append(value, search_start_pos, string::npos); - break; - } else { - new_value.append(value, search_start_pos, char_to_replace_pos - search_start_pos); - if (escape) { - new_value += "\\"; - } - new_value += replacement_characters[replace_char_ix]; - search_start_pos = char_to_replace_pos + 1; - } - } - return new_value; -} - -void StringUtils::to_lower(string& str) { - std::transform(str.cbegin(), str.cend(), str.begin(), [](unsigned char c) { - return std::tolower(c); - }); -} - -bool StringUtils::is_wildcard(char c) { - static constexpr char cWildcards[] = "?*"; - for (size_t i = 0; i < strlen(cWildcards); ++i) { - if (cWildcards[i] == c) { - return true; - } - } - return false; -} - -string StringUtils::clean_up_wildcard_search_string(string_view str) { - string cleaned_str; - - bool is_escaped = false; - auto str_end = str.cend(); - for (auto current = str.cbegin(); current != str_end;) { - auto c = *current; - if (is_escaped) { - is_escaped = false; - - if (is_wildcard(c) || '\\' == c) { - // Keep escaping if c is a wildcard character or an escape character - cleaned_str += '\\'; - } - cleaned_str += c; - ++current; - } else if ('*' == c) { - cleaned_str += c; - - // Skip over all '*' to find the next non-'*' - do { - ++current; - } while (current != str_end && '*' == *current); - } else { - if ('\\' == c) { - is_escaped = true; - } else { - cleaned_str += c; - } - ++current; - } - } - - return cleaned_str; -} - -bool StringUtils::advance_tame_to_next_match( - char const*& tame_current, - char const*& tame_bookmark, - char const* tame_end, - char const*& wild_current, - char const*& wild_bookmark -) { - auto w = *wild_current; - if ('?' != w) { - // No need to check for '*' since the caller ensures wild doesn't - // contain consecutive '*' - - // Handle escaped characters - if ('\\' == w) { - ++wild_current; - // This is safe without a bounds check since this the caller - // ensures there are no dangling escape characters - w = *wild_current; - } - - // Advance tame_current until it matches wild_current - while (true) { - if (tame_end == tame_current) { - // Wild group is longer than last group in tame, so - // can't match - // e.g. "*abc" doesn't match "zab" - return false; - } - auto t = *tame_current; - if (t == w) { - break; - } - ++tame_current; - } - } - - tame_bookmark = tame_current; - - return true; -} - -bool -StringUtils::wildcard_match_unsafe(string_view tame, string_view wild, bool case_sensitive_match) { - if (case_sensitive_match) { - return wildcard_match_unsafe_case_sensitive(tame, wild); - } else { - // We convert to lowercase (rather than uppercase) anticipating that - // callers use lowercase more frequently, so little will need to change. - string lowercase_tame(tame); - to_lower(lowercase_tame); - string lowercase_wild(wild); - to_lower(lowercase_wild); - return wildcard_match_unsafe_case_sensitive(lowercase_tame, lowercase_wild); - } -} - -/** - * The algorithm basically works as follows: - * Given a wild string "*abc*def*ghi*", it can be broken into groups of - * characters delimited by one or more '*' characters. The goal of the - * algorithm is then to determine whether the tame string contains each of - * those groups in the same order. - * - * Thus, the algorithm: - * 1. searches for the start of one of these groups in wild, - * 2. searches for a group in tame starting with the same character, and then - * 3. checks if the two match. If not, the search repeats with the next group in - * tame. - */ -bool StringUtils::wildcard_match_unsafe_case_sensitive(string_view tame, string_view wild) { - auto const tame_length = tame.length(); - auto const wild_length = wild.length(); - char const* tame_current = tame.data(); - char const* wild_current = wild.data(); - char const* tame_bookmark = nullptr; - char const* wild_bookmark = nullptr; - char const* tame_end = tame_current + tame_length; - char const* wild_end = wild_current + wild_length; - - // Handle wild or tame being empty - if (0 == wild_length) { - return 0 == tame_length; - } else { - if (0 == tame_length) { - return "*" == wild; - } - } - - char w; - char t; - bool is_escaped = false; - while (true) { - w = *wild_current; - if ('*' == w) { - ++wild_current; - if (wild_end == wild_current) { - // Trailing '*' means everything remaining in tame will match - return true; - } - - // Set wild and tame bookmarks - wild_bookmark = wild_current; - if (!advance_tame_to_next_match( - tame_current, - tame_bookmark, - tame_end, - wild_current, - wild_bookmark - )) - { - return false; - } - } else { - // Handle escaped characters - if ('\\' == w) { - is_escaped = true; - ++wild_current; - // This is safe without a bounds check since this the caller - // ensures there are no dangling escape characters - w = *wild_current; - } - - // Handle a mismatch - t = *tame_current; - if (false == ((false == is_escaped && '?' == w) || t == w)) { - if (nullptr == wild_bookmark) { - // No bookmark to return to - return false; - } - - wild_current = wild_bookmark; - tame_current = tame_bookmark + 1; - if (!advance_tame_to_next_match( - tame_current, - tame_bookmark, - tame_end, - wild_current, - wild_bookmark - )) - { - return false; - } - } - } - - ++tame_current; - ++wild_current; - - // Handle reaching the end of tame or wild - if (tame_end == tame_current) { - return (wild_end == wild_current - || ('*' == *wild_current && (wild_current + 1) == wild_end)); - } else { - if (wild_end == wild_current) { - if (nullptr == wild_bookmark) { - // No bookmark to return to - return false; - } else { - wild_current = wild_bookmark; - tame_current = tame_bookmark + 1; - if (!advance_tame_to_next_match( - tame_current, - tame_bookmark, - tame_end, - wild_current, - wild_bookmark - )) - { - return false; - } - } - } - } - } -} - -bool StringUtils::convert_string_to_int64(std::string_view raw, int64_t& converted) { - auto raw_end = raw.cend(); - auto result = std::from_chars(raw.cbegin(), raw_end, converted); - if (raw_end != result.ptr) { - return false; - } else { - return result.ec == std::errc(); - } -} - -bool StringUtils::convert_string_to_double(std::string const& raw, double& converted) { - if (raw.empty()) { - // Can't convert an empty string - return false; - } - - char const* c_str = raw.c_str(); - char* end_ptr; - // Reset errno so we can detect a new error - errno = 0; - double raw_as_double = strtod(c_str, &end_ptr); - if (ERANGE == errno || (end_ptr - c_str) < raw.length()) { - return false; - } - converted = raw_as_double; - return true; -} - void StringUtils::escape_json_string(std::string& destination, std::string_view const source) { // Escaping is implemented using this `append_unescaped_slice` approach to offer a fast path // when strings are mostly or entirely valid escaped JSON. Benchmarking shows that this offers diff --git a/components/core/src/clp_s/Utils.hpp b/components/core/src/clp_s/Utils.hpp index 7103196c72..20c8a62d96 100644 --- a/components/core/src/clp_s/Utils.hpp +++ b/components/core/src/clp_s/Utils.hpp @@ -64,22 +64,6 @@ class UriUtils { class StringUtils { public: - /** - * Checks if the given character is an alphabet - * @param c - * @return true if c is an alphabet, false otherwise - */ - static inline bool is_alphabet(char c) { - return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z'); - } - - /** - * Checks if character is a decimal (base-10) digit - * @param c - * @return true if c is a decimal digit, false otherwise - */ - static inline bool is_decimal_digit(char c) { return '0' <= c && c <= '9'; } - /** * Checks if character is a hexadecimal (base-16) digit * @param c @@ -128,114 +112,6 @@ class StringUtils { */ static bool get_bounds_of_next_var(std::string const& msg, size_t& begin_pos, size_t& end_pos); - /** - * Searches haystack starting at the given position for one of the given needles - * @param haystack - * @param needles - * @param search_start_pos - * @param needle_ix The index of the needle found - * @return The position of the match or string::npos if none - */ - static size_t find_first_of( - std::string const& haystack, - char const* needles, - size_t search_start_pos, - size_t& needle_ix - ); - - /** - * Replaces the given characters in the given value with the given replacements - * @param characters_to_escape - * @param replacement_characters - * @param value - * @param escape Whether to precede the replacement with a '\' (e.g., so that a - * line-feed character is output as "\n") - * @return The string with replacements - */ - static std::string replace_characters( - char const* characters_to_escape, - char const* replacement_characters, - std::string const& value, - bool escape - ); - - /** - * Converts a string to lowercase - * @param str - */ - static void to_lower(std::string& str); - - /** - * Cleans wildcard search string - *
    - *
  • Removes consecutive '*'
    • - *
  • Removes escaping from non-wildcard characters
    • - *
  • Removes dangling escape character from the end of the string
    • - *
- * @param str Wildcard search string to clean - * @return Cleaned wildcard search string - */ - static std::string clean_up_wildcard_search_string(std::string_view str); - - /** - * Checks if character is a wildcard - * @param c - * @return true if c is a wildcard, false otherwise - */ - static bool is_wildcard(char c); - - /** - * Same as ``wildcard_match_unsafe_case_sensitive`` except this method - * allows the caller to specify whether the match should be case sensitive. - * - * @param tame The literal string - * @param wild The wildcard string - * @param case_sensitive_match Whether to consider case when matching - * @return Whether the two strings match - */ - static bool wildcard_match_unsafe( - std::string_view tame, - std::string_view wild, - bool case_sensitive_match = true - ); - - /** - * Checks if a string matches a wildcard string. Two wildcards are currently - * supported: '*' to match 0 or more characters, and '?' to match any single - * character. Each can be escaped using a preceding '\'. Other characters which - * are escaped are treated as normal characters. - *
- * This method is optimized for performance by omitting some checks on the - * wildcard string that are unnecessary if the caller cleans up the wildcard - * string as follows: - *
    - *
  • The wildcard string should not contain consecutive '*'.
    • - *
  • The wildcard string should not contain an escape character without a - * character following it.
    • - *
- * - * @param tame The literal string - * @param wild The wildcard string - * @return Whether the two strings match - */ - static bool wildcard_match_unsafe_case_sensitive(std::string_view tame, std::string_view wild); - - /** - * Converts the given string to a 64-bit integer if possible - * @param raw - * @param converted - * @return true if the conversion was successful, false otherwise - */ - static bool convert_string_to_int64(std::string_view raw, int64_t& converted); - - /** - * Converts the given string to a double if possible - * @param raw - * @param converted - * @return true if the conversion was successful, false otherwise - */ - static bool convert_string_to_double(std::string const& raw, double& converted); - /** * Escapes a string according to JSON string escaping rules and appends the escaped string to * a buffer. The input string can be either ascii or UTF-8. @@ -253,25 +129,6 @@ class StringUtils { static void escape_json_string(std::string& destination, std::string_view const source); private: - /** - * Helper for ``wildcard_match_unsafe_case_sensitive`` to advance the - * pointer in tame to the next character which matches wild. This method - * should be inlined for performance. - * @param tame_current - * @param tame_bookmark - * @param tame_end - * @param wild_current - * @param wild_bookmark - * @return true on success, false if wild cannot match tame - */ - static inline bool advance_tame_to_next_match( - char const*& tame_current, - char const*& tame_bookmark, - char const* tame_end, - char const*& wild_current, - char const*& wild_bookmark - ); - /** * Converts a character into its two byte hexadecimal representation. * @param c diff --git a/components/core/src/clp_s/search/CMakeLists.txt b/components/core/src/clp_s/search/CMakeLists.txt index 901e4f4aee..133e08c49c 100644 --- a/components/core/src/clp_s/search/CMakeLists.txt +++ b/components/core/src/clp_s/search/CMakeLists.txt @@ -50,6 +50,7 @@ if(CLP_BUILD_CLP_S_SEARCH) clp_s::search::ast simdjson::simdjson PRIVATE + clp::string_utils clp_s::clp_dependencies clp_s::io spdlog::spdlog diff --git a/components/core/src/clp_s/search/QueryRunner.cpp b/components/core/src/clp_s/search/QueryRunner.cpp index 8f03d34929..5a0eede85b 100644 --- a/components/core/src/clp_s/search/QueryRunner.cpp +++ b/components/core/src/clp_s/search/QueryRunner.cpp @@ -3,9 +3,10 @@ #include #include +#include + #include "../../clp/type_utils.hpp" #include "../SchemaTree.hpp" -#include "../Utils.hpp" #include "ast/AndExpr.hpp" #include "ast/ColumnDescriptor.hpp" #include "ast/Expression.hpp" @@ -430,7 +431,7 @@ bool QueryRunner::evaluate_clp_string_filter( for (auto const& subquery : q->get_sub_queries()) { if (subquery.matches_logtype(id) && subquery.matches_vars(vars)) { if (subquery.wildcard_match_required()) { - matched = StringUtils::wildcard_match_unsafe( + matched = clp::string_utils::wildcard_match_unsafe( std::get(reader->extract_value(m_cur_message)), q->get_search_string(), !q->get_ignore_case() @@ -442,7 +443,7 @@ bool QueryRunner::evaluate_clp_string_filter( } } } else { - matched = StringUtils::wildcard_match_unsafe( + matched = clp::string_utils::wildcard_match_unsafe( std::get(reader->extract_value(m_cur_message)), q->get_search_string(), !q->get_ignore_case() @@ -536,7 +537,7 @@ bool QueryRunner::evaluate_array_filter_value( } break; case ondemand::json_type::string: { if (true == m_maybe_string && unresolved_tokens.size() == cur_idx - && StringUtils::wildcard_match_unsafe( + && clp::string_utils::wildcard_match_unsafe( item.get_string().value(), m_array_search_string, false == m_ignore_case @@ -676,7 +677,7 @@ bool QueryRunner::evaluate_wildcard_array_filter( if (false == m_maybe_string) { break; } - if (StringUtils::wildcard_match_unsafe( + if (clp::string_utils::wildcard_match_unsafe( item.get_string().value(), m_array_search_string, false == m_ignore_case @@ -750,7 +751,7 @@ bool QueryRunner::evaluate_wildcard_array_filter( if (false == m_maybe_string) { break; } - if (StringUtils::wildcard_match_unsafe( + if (clp::string_utils::wildcard_match_unsafe( item.get_string().value(), m_array_search_string, false == m_ignore_case diff --git a/components/core/src/clp_s/search/clp_search/Grep.cpp b/components/core/src/clp_s/search/clp_search/Grep.cpp index 40ffb456fb..7303e355e8 100644 --- a/components/core/src/clp_s/search/clp_search/Grep.cpp +++ b/components/core/src/clp_s/search/clp_search/Grep.cpp @@ -6,7 +6,8 @@ #include #include -#include "../../Utils.hpp" +#include + #include "../../VariableEncoder.hpp" #include "EncodedVariableInterpreter.hpp" @@ -422,13 +423,15 @@ std::optional Grep::process_raw_query( } // Clean-up search string - processed_search_string = StringUtils::clean_up_wildcard_search_string(processed_search_string); + processed_search_string + = clp::string_utils::clean_up_wildcard_search_string(processed_search_string); // Replace non-greedy wildcards with greedy wildcards since we currently have no support for // searching compressed files with non-greedy wildcards std::replace(processed_search_string.begin(), processed_search_string.end(), '?', '*'); // Clean-up in case any instances of "?*" or "*?" were changed into "**" - processed_search_string = StringUtils::clean_up_wildcard_search_string(processed_search_string); + processed_search_string + = clp::string_utils::clean_up_wildcard_search_string(processed_search_string); // Split search_string into tokens with wildcards vector query_tokens; @@ -538,7 +541,7 @@ bool Grep::get_bounds_of_next_potential_var( // Escape character is_escaped = true; } else { - if (StringUtils::is_wildcard(c)) { + if (clp::string_utils::is_wildcard(c)) { contains_wildcard = true; break; } @@ -570,7 +573,7 @@ bool Grep::get_bounds_of_next_potential_var( // Escape character is_escaped = true; } else { - if (StringUtils::is_wildcard(c)) { + if (clp::string_utils::is_wildcard(c)) { contains_wildcard = true; } else if (StringUtils::is_delim(c)) { // Found delimiter that's not also a wildcard @@ -578,9 +581,9 @@ bool Grep::get_bounds_of_next_potential_var( } } - if (StringUtils::is_decimal_digit(c)) { + if (clp::string_utils::is_decimal_digit(c)) { contains_decimal_digit = true; - } else if (StringUtils::is_alphabet(c)) { + } else if (clp::string_utils::is_alphabet(c)) { contains_alphabet = true; } } @@ -604,13 +607,13 @@ bool Grep::get_bounds_of_next_potential_var( if (is_escaped) { is_escaped = false; - if (StringUtils::is_alphabet(c)) { + if (clp::string_utils::is_alphabet(c)) { break; } } else if ('\\' == c) { // Escape character is_escaped = true; - } else if (StringUtils::is_wildcard(c)) { + } else if (clp::string_utils::is_wildcard(c)) { found_wildcard_before_alphabet = true; break; }