y-scope · kirkrodrigues · Dec 6, 2023 · May 31, 2023 · May 31, 2023 · Jun 1, 2023
diff --git a/.gitmodules b/.gitmodules
@@ -11,6 +11,9 @@
 [submodule "components/core/submodules/yaml-cpp"]
 	path = components/core/submodules/yaml-cpp
 	url = https://github.com/jbeder/yaml-cpp.git
+[submodule "components/core/submodules/log-surgeon"]
+	path = components/core/submodules/log-surgeon
+	url = https://github.com/y-scope/log-surgeon.git
 [submodule "components/core/submodules/boost-outcome"]
 	path = components/core/submodules/boost-outcome
 	url = https://github.com/boostorg/outcome.git
diff --git a/components/core/.clang-format b/components/core/.clang-format
@@ -68,7 +68,7 @@ IncludeBlocks: Regroup
 IncludeCategories:
   # NOTE: A header is grouped by first matching regex
   # Third-party headers. Update when adding new third-party libraries.
-  - Regex: '^<(archive|boost|catch2|date|fmt|json|mariadb|spdlog|sqlite3|yaml-cpp|zstd)'
+  - Regex: '^<(archive|boost|catch2|date|fmt|json|log_surgeon|mariadb|spdlog|sqlite3|yaml-cpp|zstd)'
     Priority: 3
   # C system headers
   - Regex: '^<.+.h>'

diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt
@@ -61,6 +61,9 @@ else()
 endif()
 message(STATUS "Building using ${CLP_LIBS_STRING} libraries")
 
+# Add log surgeon
+add_subdirectory(submodules/log-surgeon EXCLUDE_FROM_ALL)
+
 # Link against c++fs if required by the compiler being used
 set(STD_FS_LIBS "")
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
@@ -191,28 +194,6 @@ set(SOURCE_FILES_clp
         src/clp/StructuredFileToCompress.hpp
         src/clp/utils.cpp
         src/clp/utils.hpp
-        src/compressor_frontend/Constants.hpp
-        src/compressor_frontend/finite_automata/RegexAST.hpp
-        src/compressor_frontend/finite_automata/RegexAST.inc
-        src/compressor_frontend/finite_automata/RegexDFA.hpp
-        src/compressor_frontend/finite_automata/RegexDFA.inc
-        src/compressor_frontend/finite_automata/RegexNFA.hpp
-        src/compressor_frontend/finite_automata/RegexNFA.inc
-        src/compressor_frontend/finite_automata/UnicodeIntervalTree.hpp
-        src/compressor_frontend/finite_automata/UnicodeIntervalTree.inc
-        src/compressor_frontend/LALR1Parser.cpp
-        src/compressor_frontend/LALR1Parser.hpp
-        src/compressor_frontend/LALR1Parser.inc
-        src/compressor_frontend/Lexer.hpp
-        src/compressor_frontend/Lexer.inc
-        src/compressor_frontend/LogParser.cpp
-        src/compressor_frontend/LogParser.hpp
-        src/compressor_frontend/SchemaParser.cpp
-        src/compressor_frontend/SchemaParser.hpp
-        src/compressor_frontend/Token.cpp
-        src/compressor_frontend/Token.hpp
-        src/compressor_frontend/utils.cpp
-        src/compressor_frontend/utils.hpp
         src/database_utils.cpp
         src/database_utils.hpp
         src/Defs.h
@@ -283,6 +264,8 @@ set(SOURCE_FILES_clp
         src/Profiler.hpp
         src/Query.cpp
         src/Query.hpp
+        src/QueryToken.cpp
+        src/QueryToken.hpp
         src/ReaderInterface.cpp
         src/ReaderInterface.hpp
         src/spdlog_with_specializations.hpp
@@ -357,6 +340,7 @@ target_link_libraries(clp
         PRIVATE
         Boost::filesystem Boost::iostreams Boost::program_options
         fmt::fmt
+        log_surgeon::log_surgeon
         spdlog::spdlog
         ${sqlite_LIBRARY_DEPENDENCIES}
         LibArchive::LibArchive
@@ -381,26 +365,6 @@ set(SOURCE_FILES_clg
         src/clg/clg.cpp
         src/clg/CommandLineArguments.cpp
         src/clg/CommandLineArguments.hpp
-        src/compressor_frontend/Constants.hpp
-        src/compressor_frontend/finite_automata/RegexAST.hpp
-        src/compressor_frontend/finite_automata/RegexAST.inc
-        src/compressor_frontend/finite_automata/RegexDFA.hpp
-        src/compressor_frontend/finite_automata/RegexDFA.inc
-        src/compressor_frontend/finite_automata/RegexNFA.hpp
-        src/compressor_frontend/finite_automata/RegexNFA.inc
-        src/compressor_frontend/finite_automata/UnicodeIntervalTree.hpp
-        src/compressor_frontend/finite_automata/UnicodeIntervalTree.inc
-        src/compressor_frontend/LALR1Parser.cpp
-        src/compressor_frontend/LALR1Parser.hpp
-        src/compressor_frontend/LALR1Parser.inc
-        src/compressor_frontend/Lexer.hpp
-        src/compressor_frontend/Lexer.inc
-        src/compressor_frontend/SchemaParser.cpp
-        src/compressor_frontend/SchemaParser.hpp
-        src/compressor_frontend/Token.cpp
-        src/compressor_frontend/Token.hpp
-        src/compressor_frontend/utils.cpp
-        src/compressor_frontend/utils.hpp
         src/database_utils.cpp
         src/database_utils.hpp
         src/Defs.h
@@ -455,6 +419,8 @@ set(SOURCE_FILES_clg
         src/Profiler.hpp
         src/Query.cpp
         src/Query.hpp
+        src/QueryToken.cpp
+        src/QueryToken.hpp
         src/ReaderInterface.cpp
         src/ReaderInterface.hpp
         src/spdlog_with_specializations.hpp
@@ -528,6 +494,7 @@ target_link_libraries(clg
         Boost::filesystem Boost::iostreams Boost::program_options
         fmt::fmt
         KQL
+        log_surgeon::log_surgeon
         MariaDBClient::MariaDBClient
         spdlog::spdlog
         ${sqlite_LIBRARY_DEPENDENCIES}
@@ -547,26 +514,6 @@ set(SOURCE_FILES_clo
         src/clo/CommandLineArguments.hpp
         src/clo/ControllerMonitoringThread.cpp
         src/clo/ControllerMonitoringThread.hpp
-        src/compressor_frontend/Constants.hpp
-        src/compressor_frontend/finite_automata/RegexAST.hpp
-        src/compressor_frontend/finite_automata/RegexAST.inc
-        src/compressor_frontend/finite_automata/RegexDFA.hpp
-        src/compressor_frontend/finite_automata/RegexDFA.inc
-        src/compressor_frontend/finite_automata/RegexNFA.hpp
-        src/compressor_frontend/finite_automata/RegexNFA.inc
-        src/compressor_frontend/finite_automata/UnicodeIntervalTree.hpp
-        src/compressor_frontend/finite_automata/UnicodeIntervalTree.inc
-        src/compressor_frontend/LALR1Parser.cpp
-        src/compressor_frontend/LALR1Parser.hpp
-        src/compressor_frontend/LALR1Parser.inc
-        src/compressor_frontend/Lexer.hpp
-        src/compressor_frontend/Lexer.inc
-        src/compressor_frontend/SchemaParser.cpp
-        src/compressor_frontend/SchemaParser.hpp
-        src/compressor_frontend/Token.cpp
-        src/compressor_frontend/Token.hpp
-        src/compressor_frontend/utils.cpp
-        src/compressor_frontend/utils.hpp
         src/database_utils.cpp
         src/database_utils.hpp
         src/Defs.h
@@ -611,6 +558,8 @@ set(SOURCE_FILES_clo
         src/Profiler.hpp
         src/Query.cpp
         src/Query.hpp
+        src/QueryToken.cpp
+        src/QueryToken.hpp
         src/ReaderInterface.cpp
         src/ReaderInterface.hpp
         src/spdlog_with_specializations.hpp
@@ -685,6 +634,7 @@ target_link_libraries(clo
         PRIVATE
         Boost::filesystem Boost::iostreams Boost::program_options
         fmt::fmt
+        log_surgeon::log_surgeon
         msgpack-cxx
         spdlog::spdlog
         ${sqlite_LIBRARY_DEPENDENCIES}
@@ -718,28 +668,6 @@ set(SOURCE_FILES_unitTest
         src/clp/StructuredFileToCompress.hpp
         src/clp/utils.cpp
         src/clp/utils.hpp
-        src/compressor_frontend/Constants.hpp
-        src/compressor_frontend/finite_automata/RegexAST.hpp
-        src/compressor_frontend/finite_automata/RegexAST.inc
-        src/compressor_frontend/finite_automata/RegexDFA.hpp
-        src/compressor_frontend/finite_automata/RegexDFA.inc
-        src/compressor_frontend/finite_automata/RegexNFA.hpp
-        src/compressor_frontend/finite_automata/RegexNFA.inc
-        src/compressor_frontend/finite_automata/UnicodeIntervalTree.hpp
-        src/compressor_frontend/finite_automata/UnicodeIntervalTree.inc
-        src/compressor_frontend/LALR1Parser.cpp
-        src/compressor_frontend/LALR1Parser.hpp
-        src/compressor_frontend/LALR1Parser.inc
-        src/compressor_frontend/Lexer.hpp
-        src/compressor_frontend/Lexer.inc
-        src/compressor_frontend/LogParser.cpp
-        src/compressor_frontend/LogParser.hpp
-        src/compressor_frontend/SchemaParser.cpp
-        src/compressor_frontend/SchemaParser.hpp
-        src/compressor_frontend/Token.cpp
-        src/compressor_frontend/Token.hpp
-        src/compressor_frontend/utils.cpp
-        src/compressor_frontend/utils.hpp
         src/database_utils.cpp
         src/database_utils.hpp
         src/Defs.h
@@ -827,6 +755,8 @@ set(SOURCE_FILES_unitTest
         src/Profiler.hpp
         src/Query.cpp
         src/Query.hpp
+        src/QueryToken.cpp
+        src/QueryToken.hpp
         src/ReaderInterface.cpp
         src/ReaderInterface.hpp
         src/spdlog_with_specializations.hpp
@@ -919,6 +849,7 @@ target_link_libraries(unitTest
         PRIVATE
         Boost::filesystem Boost::iostreams Boost::program_options
         fmt::fmt
+        log_surgeon::log_surgeon
         LibArchive::LibArchive
         MariaDBClient::MariaDBClient
         spdlog::spdlog

diff --git a/components/core/README-Schema.md b/components/core/README-Schema.md
@@ -17,7 +17,7 @@ delimiters: \t\r\n:,!;%
 timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3}){0,1}
 timestamp:\[\d{8}\-\d{2}:\d{2}:\d{2}\]
 int:\-{0,1}[0-9]+
-double:\-{0,1}[0-9]+\.[0-9]+
+float:\-{0,1}[0-9]+\.[0-9]+
 
 // Custom variables
 hex:[a-fA-F]+
@@ -49,7 +49,7 @@ equals:.*=.*[a-zA-Z0-9].*
   start of the file then a newline is used to indicate the beginning of a new
   log message. Timestamp patterns are not matched midline and are not stored as
   dictionary variables as they may contain delimiters.
-* `int` and `double` are keywords. These are encoded specially for compression
+* `int` and `float` are keywords. These are encoded specially for compression
   performance.
 
 ## Supported Regex

diff --git a/components/core/cmake/utils.cmake b/components/core/cmake/utils.cmake
@@ -41,9 +41,14 @@ set(SOURCE_FILES_make-dictionaries-readable
         ${CMAKE_CURRENT_SOURCE_DIR}/submodules/date/include/date/date.h
         )
 add_executable(make-dictionaries-readable ${SOURCE_FILES_make-dictionaries-readable})
+target_include_directories(make-dictionaries-readable
+                           PRIVATE
+                           ${CMAKE_SOURCE_DIR}/submodules
+                           )
 target_link_libraries(make-dictionaries-readable
         PRIVATE
         Boost::filesystem Boost::iostreams Boost::program_options
+        log_surgeon::log_surgeon
         spdlog::spdlog
         ZStd::ZStd
         )

diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt
@@ -9,9 +9,9 @@ timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3}){0,1}
 // E.g. [20150131-15:50:45]
 timestamp:\[\d{8}\-\d{2}:\d{2}:\d{2}\]
 
-// Specially-encoded variables (using the `int` and `double` keywords)
+// Specially-encoded variables (using the `int` and `float` keywords)
 int:\-{0,1}[0-9]+
-double:\-{0,1}[0-9]+\.[0-9]+
+float:\-{0,1}[0-9]+\.[0-9]+
 
 // Dictionary variables
 hex:[a-fA-F]+

diff --git a/components/core/src/FileReader.cpp b/components/core/src/FileReader.cpp
@@ -87,7 +87,7 @@ void FileReader::open (const string& path) {
     ErrorCode error_code = try_open(path);
     if (ErrorCode_Success != error_code) {
         if (ErrorCode_FileNotFound == error_code) {
-            throw "File not found: " + boost::filesystem::weakly_canonical(path).string() + "\n"; 
+            throw "File not found: " + boost::filesystem::weakly_canonical(path).string() + "\n";
         } else {
             throw OperationFailed(error_code, __FILENAME__, __LINE__);
         }

diff --git a/components/core/src/GlobalMetadataDBConfig.cpp b/components/core/src/GlobalMetadataDBConfig.cpp
@@ -4,7 +4,7 @@
 #include <fmt/core.h>
 
 // yaml-cpp
-#include <yaml-cpp/include/yaml-cpp/yaml.h>
+#include <yaml-cpp/yaml.h>
 
 using std::exception;
 using std::invalid_argument;