feat(log-surgeon)!: Support using a single capture group in schema rules to achieve parity with the heuristic parser.

components/core/config/schemas.txt

@@ -16,4 +16,4 @@ float:\-{0,1}[0-9]+\.[0-9]+
 // Dictionary variables
 hex:[a-fA-F]+
 hasNumber:.*\d.*
-equals:.*=.*[a-zA-Z0-9].*
+equals:.*=(?<var>.*[a-zA-Z0-9].*)

components/core/src/clp/GrepCore.cpp

@@ -257,9 +257,9 @@ bool GrepCore::get_bounds_of_next_potential_var(
                         return false;
                     }
                     search_token = SearchToken{token.value()};
-                    search_token.m_type_ids_set.insert(search_token.m_type_ids_ptr->at(0));
+                    search_token.m_type_ids_set.insert(search_token.get_type_ids()->at(0));
                 }
-                auto const& type = search_token.m_type_ids_ptr->at(0);
+                auto const& type = search_token.get_type_ids()->at(0);
                 if (type != static_cast<int>(log_surgeon::SymbolId::TokenUncaughtString)
                     && type != static_cast<int>(log_surgeon::SymbolId::TokenEnd))
                 {

components/core/src/clp/LogTypeDictionaryEntry.cpp

@@ -1,16 +1,20 @@
 #include "LogTypeDictionaryEntry.hpp"
 
-#include "EncodedVariableInterpreter.hpp"
-#include "ir/parsing.hpp"
-#include "ir/types.hpp"
-#include "type_utils.hpp"
-#include "Utils.hpp"
+#include <cstddef>
+#include <cstdint>
+#include <string>
+#include <string_view>
 
+#include <clp/EncodedVariableInterpreter.hpp>
+#include <clp/ir/parsing.hpp>
+#include <clp/ir/types.hpp>
+#include <clp/type_utils.hpp>
+
+namespace clp {
 using clp::ir::VariablePlaceholder;
 using std::string;
 using std::string_view;
 
-namespace clp {
 size_t LogTypeDictionaryEntry::get_placeholder_info(
         size_t placeholder_ix,
         VariablePlaceholder& placeholder
@@ -34,7 +38,7 @@ size_t LogTypeDictionaryEntry::get_data_size() const {
 }
 
 void LogTypeDictionaryEntry::add_constant(
-        std::string_view value_containing_constant,
+        string_view value_containing_constant,
         size_t begin_pos,
         size_t length
 ) {
@@ -62,35 +66,37 @@ void LogTypeDictionaryEntry::add_escape() {
     ++m_num_escaped_placeholders;
 }
 
-bool LogTypeDictionaryEntry::parse_next_var(
-        std::string_view msg,
+auto LogTypeDictionaryEntry::add_static_text(string_view static_text) -> void {
+    ir::append_constant_to_logtype(
+            static_text,
+            [&]([[maybe_unused]] string_view constant,
+                [[maybe_unused]] size_t char_to_escape_pos,
+                [[maybe_unused]] string& logtype) -> void { add_escape(); },
+            m_value
+    );
+}
+
+auto LogTypeDictionaryEntry::parse_next_var(
+        string_view msg,
         size_t& var_begin_pos,
         size_t& var_end_pos,
-        std::string_view& var
-) {
-    auto last_var_end_pos = var_end_pos;
-    // clang-format off
-    auto escape_handler = [&](
-            [[maybe_unused]] string_view constant,
-            [[maybe_unused]] size_t char_to_escape_pos,
-            string& logtype
-    ) -> void {
-        m_placeholder_positions.push_back(logtype.size());
-        ++m_num_escaped_placeholders;
-        logtype += enum_to_underlying_type(VariablePlaceholder::Escape);
-    };
-    // clang-format on
+        string_view& var
+) -> bool {
+    auto const last_var_end_pos{var_end_pos};
+    auto escape_handler = [&]([[maybe_unused]] string_view constant,
+                              [[maybe_unused]] size_t char_to_escape_pos,
+                              [[maybe_unused]] string& logtype) -> void { add_escape(); };
     if (ir::get_bounds_of_next_var(msg, var_begin_pos, var_end_pos)) {
         // Append to log type: from end of last variable to start of current variable
-        auto constant = msg.substr(last_var_end_pos, var_begin_pos - last_var_end_pos);
+        auto const constant{msg.substr(last_var_end_pos, var_begin_pos - last_var_end_pos)};
         ir::append_constant_to_logtype(constant, escape_handler, m_value);
 
         var = msg.substr(var_begin_pos, var_end_pos - var_begin_pos);
         return true;
     }
     if (last_var_end_pos < msg.length()) {
         // Append to log type: from end of last variable to end
-        auto constant = msg.substr(last_var_end_pos, msg.length() - last_var_end_pos);
+        auto const constant{msg.substr(last_var_end_pos, msg.length() - last_var_end_pos)};
         ir::append_constant_to_logtype(constant, escape_handler, m_value);
     }
 

components/core/src/clp/LogTypeDictionaryEntry.hpp

@@ -92,6 +92,11 @@ class LogTypeDictionaryEntry : public DictionaryEntry<logtype_dictionary_id_t> {
      */
     void add_escape();
 
+    /**
+     * Adds static text to the logtype escaping any placeholder characters.
+     */
+    auto add_static_text(std::string_view static_text) -> void;
+
     /**
      * Parses next variable from a message, constructing the constant part of the message's logtype
      * as well
@@ -103,12 +108,12 @@ class LogTypeDictionaryEntry : public DictionaryEntry<logtype_dictionary_id_t> {
      * @param var
      * @return true if another variable was found, false otherwise
      */
-    bool parse_next_var(
+    auto parse_next_var(
             std::string_view msg,
             size_t& var_begin_pos,
             size_t& var_end_pos,
             std::string_view& var
-    );
+    ) -> bool;
 
     /**
      * Reserves space for a constant of the given length

components/core/src/clp/LogTypeDictionaryEntryReq.hpp

@@ -24,7 +24,8 @@ concept LogTypeDictionaryEntryReq = requires(
         std::string_view& parsed_var_ref,
         size_t begin_pos,
         size_t placeholder_idx,
-        ir::VariablePlaceholder& placeholder_ref
+        ir::VariablePlaceholder& placeholder_ref,
+        std::string_view static_text
 ) {
     /**
      * Clears all internal state.
@@ -74,6 +75,16 @@ concept LogTypeDictionaryEntryReq = requires(
      */
     { entry.add_dictionary_var() } -> std::same_as<void>;
 
+    /**
+     * Adds an escape character
+     */
+    { entry.add_escape() } -> std::same_as<void>;
+
+    /**
+     * Adds static text to the logtype escaping any placeholder characters.
+     */
+    { entry.add_static_text(static_text) } -> std::same_as<void>;
+
     /**
      * @return The constant part of the logtype.
      */

components/core/src/clp/Utils.cpp

@@ -8,6 +8,7 @@
 #include <iostream>
 #include <memory>
 #include <set>
+#include <string>
 
 #include <boost/algorithm/string.hpp>
 #include <boost/lexical_cast.hpp>
@@ -140,10 +141,8 @@ load_lexer_from_file(std::string const& schema_file_path, log_surgeon::lexers::B
     lexer.m_symbol_id[log_surgeon::cTokenInt] = static_cast<int>(log_surgeon::SymbolId::TokenInt);
     lexer.m_symbol_id[log_surgeon::cTokenFloat]
             = static_cast<int>(log_surgeon::SymbolId::TokenFloat);
-    lexer.m_symbol_id[log_surgeon::cTokenFirstTimestamp]
-            = static_cast<int>(log_surgeon::SymbolId::TokenFirstTimestamp);
-    lexer.m_symbol_id[log_surgeon::cTokenNewlineTimestamp]
-            = static_cast<int>(log_surgeon::SymbolId::TokenNewlineTimestamp);
+    lexer.m_symbol_id[log_surgeon::cTokenHeader]
+            = static_cast<int>(log_surgeon::SymbolId::TokenHeader);
     // cTokenNewline is not added in schema_vars and can be explicitly added as '\n' to catch the
     // end of non-timestamped log messages
     lexer.m_symbol_id[log_surgeon::cTokenNewline]
@@ -155,10 +154,8 @@ load_lexer_from_file(std::string const& schema_file_path, log_surgeon::lexers::B
     lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolId::TokenInt)] = log_surgeon::cTokenInt;
     lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolId::TokenFloat)]
             = log_surgeon::cTokenFloat;
-    lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolId::TokenFirstTimestamp)]
-            = log_surgeon::cTokenFirstTimestamp;
-    lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolId::TokenNewlineTimestamp)]
-            = log_surgeon::cTokenNewlineTimestamp;
+    lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolId::TokenHeader)]
+            = log_surgeon::cTokenHeader;
     lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolId::TokenNewline)]
             = log_surgeon::cTokenNewline;
 
@@ -187,6 +184,18 @@ load_lexer_from_file(std::string const& schema_file_path, log_surgeon::lexers::B
     for (std::unique_ptr<log_surgeon::ParserAST> const& parser_ast : schema_ast->m_schema_vars) {
         auto* rule = dynamic_cast<log_surgeon::SchemaVarAST*>(parser_ast.get());
 
+        // Currently, we only support at most a single capture group in each variable. If a capture
+        // group is present its match will be treated as the variable rather than the full match.
+        auto const num_captures = rule->m_regex_ptr->get_subtree_positive_captures().size();
+        if (1 < num_captures) {
+            throw std::runtime_error(
+                    schema_file_path + ":" + std::to_string(rule->m_line_num + 1)
+                    + ": error: the schema rule '" + rule->m_name
+                    + "' has a regex pattern containing > 1 capture groups (found "
+                    + std::to_string(num_captures) + ").\n"
+            );
+        }
+
         if ("timestamp" == rule->m_name) {
             continue;
         }