feat: separate the concept of captures and tags; lexer now tracks mapping from variables to capture to tags to registers.

CMakeLists.txt

@@ -93,17 +93,17 @@ set(SOURCE_FILES
     src/log_surgeon/SchemaParser.hpp
     src/log_surgeon/Token.cpp
     src/log_surgeon/Token.hpp
-    src/log_surgeon/finite_automata/PrefixTree.cpp
-    src/log_surgeon/finite_automata/PrefixTree.hpp
-    src/log_surgeon/finite_automata/RegexAST.hpp
+    src/log_surgeon/finite_automata/Capture.hpp
     src/log_surgeon/finite_automata/Dfa.hpp
     src/log_surgeon/finite_automata/DfaState.hpp
     src/log_surgeon/finite_automata/DfaStatePair.hpp
     src/log_surgeon/finite_automata/Nfa.hpp
     src/log_surgeon/finite_automata/NfaState.hpp
+    src/log_surgeon/finite_automata/PrefixTree.cpp
+    src/log_surgeon/finite_automata/PrefixTree.hpp
+    src/log_surgeon/finite_automata/RegexAST.hpp
     src/log_surgeon/finite_automata/RegisterHandler.hpp
     src/log_surgeon/finite_automata/StateType.hpp
-    src/log_surgeon/finite_automata/Tag.hpp
     src/log_surgeon/finite_automata/TaggedTransition.hpp
     src/log_surgeon/finite_automata/UnicodeIntervalTree.hpp
     src/log_surgeon/finite_automata/UnicodeIntervalTree.tpp

src/log_surgeon/Lexer.hpp

@@ -15,6 +15,7 @@
 #include <log_surgeon/finite_automata/DfaState.hpp>
 #include <log_surgeon/finite_automata/Nfa.hpp>
 #include <log_surgeon/finite_automata/RegexAST.hpp>
+#include <log_surgeon/finite_automata/RegisterHandler.hpp>
 #include <log_surgeon/LexicalRule.hpp>
 #include <log_surgeon/ParserInputBuffer.hpp>
 #include <log_surgeon/Token.hpp>
@@ -23,6 +24,10 @@ namespace log_surgeon {
 template <typename TypedNfaState, typename TypedDfaState>
 class Lexer {
 public:
+    using register_id_t = finite_automata::RegisterHandler::register_id_t;
+    using symbol_id_t = uint32_t;
+    using tag_id_t = finite_automata::tag_id_t;
+
     static inline std::vector<uint32_t> const cTokenEndTypes = {(uint32_t)SymbolId::TokenEnd};
     static inline std::vector<uint32_t> const cTokenUncaughtStringTypes
             = {(uint32_t)SymbolId::TokenUncaughtString};
@@ -51,7 +56,8 @@ class Lexer {
     auto get_rule(uint32_t variable_id) -> finite_automata::RegexAST<TypedNfaState>*;
 
     /**
-     * Generate DFA for lexer
+     * Generate DFA for lexer.
+     * @throw std::invalid_argument if `m_rules` contains multipe captures with the same name.
      */
     auto generate() -> void;
 
@@ -122,8 +128,48 @@ class Lexer {
         return m_dfa;
     }
 
-    std::unordered_map<std::string, uint32_t> m_symbol_id;
-    std::unordered_map<uint32_t, std::string> m_id_symbol;
+    [[nodiscard]] auto get_capture_ids_for_var_id(symbol_id_t const var_id
+    ) const -> std::optional<std::vector<symbol_id_t>> {
+        auto const capture_ids{m_var_id_to_capture_ids.find(var_id)};
+        if (m_var_id_to_capture_ids.end() == capture_ids) {
+            return std::nullopt;
+        }
+        return capture_ids->second;
+    }
+
+    [[nodiscard]] auto get_tag_ids_for_capture_id(symbol_id_t const capture_id
+    ) const -> std::optional<std::pair<tag_id_t, tag_id_t>> {
+        auto const tag_ids{m_capture_id_to_tag_ids.find(capture_id)};
+        if (m_capture_id_to_tag_ids.end() == tag_ids) {
+            return std::nullopt;
+        }
+        return tag_ids->second;
+    }
+
+    [[nodiscard]] auto get_register_for_tag_id(tag_id_t const tag_id
+    ) const -> std::optional<register_id_t> {
+        auto const it{m_tag_to_register_id.find(tag_id)};
+        if (m_tag_to_register_id.end() == it) {
+            return std::nullopt;
+        }
+        return it->second;
+    }
+
+    [[nodiscard]] auto get_registers_for_capture(symbol_id_t capture_id
+    ) const -> std::optional<std::pair<register_id_t, register_id_t>> {
+        auto const tag_ids{get_tag_ids_for_capture_id(capture_id)};
+        if (tag_ids.has_value()) {
+            auto const start_reg{get_register_for_tag_id(tag_ids.value().first())};
+            auto const end_reg{get_register_for_tag_id(tag_ids.value().second())};
+            if (start_reg.has_value() && end_reg.has_value()) {
+                return std::make_pair(start_reg.value(), end_reg.value());
+            }
+        }
+        return std::nullopt;
+    }
+
+    std::unordered_map<std::string, symbol_id_t> m_symbol_id;
+    std::unordered_map<symbol_id_t, std::string> m_id_symbol;
 
 private:
     /**
@@ -148,6 +194,9 @@ class Lexer {
     std::unique_ptr<finite_automata::Dfa<TypedDfaState>> m_dfa;
     bool m_asked_for_more_data{false};
     TypedDfaState const* m_prev_state{nullptr};
+    std::unordered_map<symbol_id_t, std::vector<symbol_id_t>> m_var_id_to_capture_ids;
+    std::unordered_map<symbol_id_t, std::pair<tag_id_t, tag_id_t>> m_capture_id_to_tag_ids;
+    std::unordered_map<tag_id_t, register_id_t> m_tag_to_register_id;
 };
 
 namespace lexers {

src/log_surgeon/Lexer.tpp

@@ -4,6 +4,7 @@
 #include <cassert>
 #include <memory>
 #include <stack>
+#include <stdexcept>
 #include <string>
 #include <vector>
 
@@ -358,17 +359,17 @@ void Lexer<TypedNfaState, TypedDfaState>::add_delimiters(std::vector<uint32_t> c
 
 template <typename TypedNfaState, typename TypedDfaState>
 void Lexer<TypedNfaState, TypedDfaState>::add_rule(
-        uint32_t const& id,
+        symbol_id_t const& var_id,
         std::unique_ptr<finite_automata::RegexAST<TypedNfaState>> rule
 ) {
-    m_rules.emplace_back(id, std::move(rule));
+    m_rules.emplace_back(var_id, std::move(rule));
 }
 
 template <typename TypedNfaState, typename TypedDfaState>
-auto Lexer<TypedNfaState, TypedDfaState>::get_rule(uint32_t const variable_id
+auto Lexer<TypedNfaState, TypedDfaState>::get_rule(symbol_id_t const var_id
 ) -> finite_automata::RegexAST<TypedNfaState>* {
     for (auto const& rule : m_rules) {
-        if (rule.get_variable_id() == variable_id) {
+        if (rule.get_variable_id() == var_id) {
             return rule.get_regex();
         }
     }
@@ -377,8 +378,30 @@ auto Lexer<TypedNfaState, TypedDfaState>::get_rule(uint32_t const variable_id
 
 template <typename TypedNfaState, typename TypedDfaState>
 void Lexer<TypedNfaState, TypedDfaState>::generate() {
-    finite_automata::Nfa<TypedNfaState> nfa{std::move(m_rules)};
-    // TODO: DFA ignores tags. E.g., treats "capture:user=(?<user_id>\d+)" as "capture:user=\d+"
+    for (auto const& rule : m_rules) {
+        for (auto* capture : rule.get_captures()) {
+            std::string const capture_name{capture->get_name()};
+            symbol_id_t capture_id{0};
+            if (m_symbol_id.find(capture_name) == m_symbol_id.end()) {
+                capture_id = m_symbol_id.size();
+                m_symbol_id[capture_name] = capture_id;
+                m_id_symbol[capture_id] = capture_name;
+            } else {
+                throw std::invalid_argument("`m_rules` contains capture names that are not unique."
+                );
+            }
+            m_var_id_to_capture_ids[rule.get_variable_id()].push_back(capture_id);
+        }
+    }
+
+    finite_automata::Nfa<TypedNfaState> nfa{m_rules};
+    for (auto const& [capture, tag_ids] : nfa.get_capture_to_tag_ids()) {
+        std::string capture_name{capture->get_name()};
+        auto capture_id{m_symbol_id[capture_name]};
+        m_capture_id_to_tag_ids.emplace(capture_id, tag_ids);
+    }
+
+    // TODO: DFA ignores captures. E.g., treats "capture:user=(?<user_id>\d+)" as "capture:user=\d+"
     m_dfa = std::make_unique<finite_automata::Dfa<TypedDfaState>>(std::move(nfa));
     auto const* state = m_dfa->get_root();
     for (uint32_t i = 0; i < cSizeOfByte; i++) {

src/log_surgeon/LexicalRule.hpp

@@ -23,6 +23,10 @@ class LexicalRule {
      */
     auto add_to_nfa(finite_automata::Nfa<TypedNfaState>* nfa) const -> void;
 
+    [[nodiscard]] auto get_captures() const -> std::vector<finite_automata::Capture const*> {
+        return m_regex->get_subtree_positive_captures();
+    }
+
     [[nodiscard]] auto get_variable_id() const -> uint32_t { return m_variable_id; }
 
     [[nodiscard]] auto get_regex() const -> finite_automata::RegexAST<TypedNfaState>* {

src/log_surgeon/SchemaParser.cpp

@@ -9,8 +9,8 @@
 
 #include <log_surgeon/Constants.hpp>
 #include <log_surgeon/FileReader.hpp>
+#include <log_surgeon/finite_automata/Capture.hpp>
 #include <log_surgeon/finite_automata/RegexAST.hpp>
-#include <log_surgeon/finite_automata/Tag.hpp>
 #include <log_surgeon/Lalr1Parser.hpp>
 #include <log_surgeon/Lexer.hpp>
 #include <log_surgeon/utils.hpp>
@@ -167,7 +167,7 @@ static auto regex_capture_rule(NonTerminal const* m) -> std::unique_ptr<ParserAS
     auto& r6 = m->non_terminal_cast(5)->get_parser_ast()->get<unique_ptr<RegexASTByte>>();
     return std::make_unique<ParserValueRegex>(make_unique<RegexASTCaptureByte>(
             std::move(r6),
-            std::make_unique<finite_automata::Tag>(r4->m_name)
+            std::make_unique<finite_automata::Capture>(r4->m_name)
     ));
 }
 
@@ -202,7 +202,7 @@ static auto regex_or_rule(NonTerminal* m) -> unique_ptr<ParserAST> {
 static auto regex_match_zero_or_more_rule(NonTerminal* m) -> unique_ptr<ParserAST> {
     auto& r1 = m->non_terminal_cast(0)->get_parser_ast()->get<unique_ptr<RegexASTByte>>();
 
-    // To handle negative tags we treat `R*` as `R+ | ∅`.
+    // To handle negative captures we treat `R*` as `R+ | ∅`.
     return make_unique<ParserValueRegex>(make_unique<RegexASTOrByte>(
             make_unique<RegexASTEmptyByte>(),
             make_unique<RegexASTMultiplicationByte>(std::move(r1), 1, 0)
@@ -248,7 +248,7 @@ static auto regex_match_range_rule(NonTerminal* m) -> unique_ptr<ParserAST> {
     auto& r1 = m->non_terminal_cast(0)->get_parser_ast()->get<unique_ptr<RegexASTByte>>();
 
     if (0 == min) {
-        // To handle negative tags we treat `R*` as `R+ | ∅`.
+        // To handle negative captures we treat `R*` as `R+ | ∅`.
         return make_unique<ParserValueRegex>(make_unique<RegexASTOrByte>(
                 make_unique<RegexASTEmptyByte>(),
                 make_unique<RegexASTMultiplicationByte>(std::move(r1), 1, max)

src/log_surgeon/finite_automata/Tag.hpp → src/log_surgeon/finite_automata/Capture.hpp

@@ -1,14 +1,14 @@
-#ifndef LOG_SURGEON_FINITE_AUTOMATA_TAG
-#define LOG_SURGEON_FINITE_AUTOMATA_TAG
+#ifndef LOG_SURGEON_FINITE_AUTOMATA_CAPTURE
+#define LOG_SURGEON_FINITE_AUTOMATA_CAPTURE
 
 #include <string>
 #include <string_view>
 #include <utility>
 
 namespace log_surgeon::finite_automata {
-class Tag {
+class Capture {
 public:
-    explicit Tag(std::string name) : m_name{std::move(name)} {}
+    explicit Capture(std::string name) : m_name{std::move(name)} {}
 
     [[nodiscard]] auto get_name() const -> std::string_view { return m_name; }
 
@@ -17,4 +17,4 @@ class Tag {
 };
 }  // namespace log_surgeon::finite_automata
 
-#endif  // LOG_SURGEON_FINITE_AUTOMATA_TAG
+#endif  // LOG_SURGEON_FINITE_AUTOMATA_CAPTURE

src/log_surgeon/finite_automata/Dfa.hpp

@@ -2,12 +2,16 @@
 #define LOG_SURGEON_FINITE_AUTOMATA_DFA_HPP
 
 #include <cstdint>
+#include <map>
 #include <memory>
 #include <set>
+#include <stack>
 #include <vector>
 
+#include <log_surgeon/Constants.hpp>
 #include <log_surgeon/finite_automata/DfaStatePair.hpp>
 #include <log_surgeon/finite_automata/Nfa.hpp>
+#include <log_surgeon/finite_automata/RegisterHandler.hpp>
 
 namespace log_surgeon::finite_automata {
 template <typename TypedDfaState>
@@ -38,6 +42,7 @@ class Dfa {
 
 private:
     std::vector<std::unique_ptr<TypedDfaState>> m_states;
+    RegisterHandler m_register_handler;
 };
 
 template <typename TypedDfaState>
@@ -74,7 +79,7 @@ Dfa<TypedDfaState>::Dfa(Nfa<TypedNfaState> nfa) {
         }
         auto next_dfa_state
                 = [&dfa_states, &create_dfa_state](StateSet const& set) -> TypedDfaState* {
-            TypedDfaState* state;
+            TypedDfaState* state{nullptr};
             auto it = dfa_states.find(set);
             if (it == dfa_states.end()) {
                 state = create_dfa_state(set);