build(deps): Migrate core's submodule-based dependencies to task-based installation.

.github/workflows/clp-core-build-macos.yaml

@@ -13,7 +13,7 @@ on:
       - "components/core/tools/scripts/deps-download/**"
       - "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
       - "taskfile.yaml"
-      - "taskfiles/deps.yaml"
+      - "taskfiles/**"
       - "tools/scripts/deps-download/**"
   push:
     paths:
@@ -27,7 +27,7 @@ on:
       - "components/core/tools/scripts/deps-download/**"
       - "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
       - "taskfile.yaml"
-      - "taskfiles/deps.yaml"
+      - "taskfiles/**"
       - "tools/scripts/deps-download/**"
   schedule:
     # Run daily at 00:15 UTC (the 15 is to avoid periods of high load)

.github/workflows/clp-core-build.yaml

@@ -9,7 +9,7 @@ on:
       - ".gitmodules"
       - "components/core/**"
       - "taskfile.yaml"
-      - "taskfiles/deps.yaml"
+      - "taskfiles/**"
       - "tools/scripts/deps-download/**"
       - "!components/core/tools/scripts/lib_install/macos/**"
   push:
@@ -20,7 +20,7 @@ on:
       - ".gitmodules"
       - "components/core/**"
       - "taskfile.yaml"
-      - "taskfiles/deps.yaml"
+      - "taskfiles/**"
       - "tools/scripts/deps-download/**"
       - "!components/core/tools/scripts/lib_install/macos/**"
   schedule:
@@ -88,7 +88,7 @@ jobs:
               - "components/core/tests/**"
               - "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
               - "taskfile.yaml"
-              - "taskfiles/deps.yaml"
+              - "taskfiles/**"
               - "tools/scripts/deps-download/**"
 
   centos-stream-9-deps-image:

.gitmodules

@@ -1,34 +1,3 @@
-[submodule "components/core/submodules/Catch2"]
-	path = components/core/submodules/Catch2
-	url = https://github.com/catchorg/Catch2.git
-[submodule "components/core/submodules/date"]
-	path = components/core/submodules/date
-	url = https://github.com/HowardHinnant/date.git
-[submodule "components/core/submodules/json"]
-	path = components/core/submodules/json
-	url = https://github.com/nlohmann/json.git
-	shallow = true
-[submodule "components/core/submodules/yaml-cpp"]
-	path = components/core/submodules/yaml-cpp
-	url = https://github.com/jbeder/yaml-cpp.git
-[submodule "components/core/submodules/log-surgeon"]
-	path = components/core/submodules/log-surgeon
-	url = https://github.com/y-scope/log-surgeon.git
-[submodule "components/core/submodules/simdjson"]
-	path = components/core/submodules/simdjson
-	url = https://github.com/simdjson/simdjson.git
-[submodule "components/core/submodules/abseil-cpp"]
-	path = components/core/submodules/abseil-cpp
-	url = https://github.com/abseil/abseil-cpp.git
 [submodule "tools/yscope-dev-utils"]
 	path = tools/yscope-dev-utils
 	url = https://github.com/y-scope/yscope-dev-utils.git
-[submodule "components/core/submodules/outcome"]
-	path = components/core/submodules/outcome
-	url = https://github.com/ned14/outcome.git
-[submodule "components/core/submodules/utfcpp"]
-	path = components/core/submodules/utfcpp
-	url = https://github.com/nemtrif/utfcpp.git
-[submodule "components/core/submodules/ystdlib-cpp"]
-	path = components/core/submodules/ystdlib-cpp
-	url = https://github.com/y-scope/ystdlib-cpp.git

components/core/.clang-format

@@ -4,9 +4,9 @@ IncludeCategories:
   # NOTE: A header is grouped by first matching regex
   # Library headers. Update when adding new libraries.
   # NOTE: clang-format retains leading white-space on a line in violation of the YAML spec.
-  - Regex: "<(absl|antlr4|archive|boost|bsoncxx|catch2|curl|date|fmt|json|log_surgeon|lzma|mongocxx\
-|msgpack|mysql|openssl|outcome|regex_utils|simdjson|spdlog|sqlite3|string_utils|yaml-cpp|ystdlib\
-|zstd)"
+  - Regex: "<(absl|antlr4|archive|boost|bsoncxx|catch2|curl|date|fmt|log_surgeon|lzma|mongocxx\
+|msgpack|mysql|nlohmann|openssl|outcome|regex_utils|simdjson|spdlog|sqlite3|string_utils|yaml-cpp\
+|ystdlib|zstd)"
     Priority: 3
   # C system headers
   - Regex: "^<.+\\.h>"

components/core/.gitignore

@@ -1,3 +1 @@
 build/**
-submodules/sqlite3/*
-third-party/**

components/core/CMakeLists.txt

@@ -96,6 +96,28 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
     endif ()
 endif ()
 
+if(PROJECT_IS_TOP_LEVEL)
+    # Include dependency settings if the project isn't being included as a subproject.
+    # NOTE: We mark the file optional because if the user happens to have the dependencies
+    # installed, this file isn't necessary.
+    include("${CMAKE_SOURCE_DIR}/../../build/deps/core/cmake-settings/all.cmake"
+            OPTIONAL
+            RESULT_VARIABLE CLP_DEPS_SETTINGS_FILE_PATH
+    )
+
+    if(NOT CLP_DEPS_SETTINGS_FILE_PATH STREQUAL "NOTFOUND")
+        # Set CMP0144 since our minimum required CMake version is less than 3.27.
+        if(POLICY "CMP0144")
+            cmake_policy(SET "CMP0144" "NEW")
+        endif()
+    endif()
+endif()
+
+find_package(absl REQUIRED)
+if (absl_FOUND)
+    message(STATUS "Found absl ${absl_VERSION}")
+endif()
+
 # Find and setup ANTLR Library
 # We build and link to the static library
 find_package(ANTLR REQUIRED)
@@ -116,6 +138,16 @@ else()
     message(FATAL_ERROR "Could not find ${CLP_LIBS_STRING} libraries for Boost")
 endif()
 
+find_package(Catch2 REQUIRED)
+if (Catch2_FOUND)
+    message(STATUS "Found Catch2 ${Catch2_VERSION}")
+endif()
+
+find_package(date REQUIRED)
+if (date_FOUND)
+    message(STATUS "Found date ${date_VERSION}")
+endif()
+
 # Find and setup fmt
 # NOTE:
 # - We only try to link to the static library
@@ -127,6 +159,21 @@ else()
     message(FATAL_ERROR "Could not find static libraries for fmt")
 endif()
 
+find_package(log_surgeon REQUIRED)
+if(log_surgeon_FOUND)
+    message(STATUS "Found log_surgeon ${log_surgeon_VERSION}")
+endif()
+
+find_package(nlohmann_json REQUIRED)
+if(nlohmann_json_FOUND)
+    message(STATUS "Found nlohmann_json ${nlohmann_json_VERSION}")
+endif()
+
+find_package(simdjson REQUIRED)
+if(simdjson_FOUND)
+    message(STATUS "Found simdjson ${simdjson_VERSION}")
+endif()
+
 # Find and setup spdlog
 if(CLP_USE_STATIC_LIBS)
     # NOTE: On some Linux distributions (e.g. Ubuntu), the spdlog package only contains a dynamic
@@ -175,9 +222,6 @@ else ()
     message(FATAL_ERROR "OpenSSL not found")
 endif ()
 
-# Add log surgeon
-add_subdirectory(submodules/log-surgeon EXCLUDE_FROM_ALL)
-
 # Find and setup MariaDBClient library
 if(CLP_USE_STATIC_LIBS)
     # NOTE: We can't statically link to MariaDBClient since it's GPL
@@ -212,21 +256,16 @@ else()
     message(FATAL_ERROR "Could not find msgpack-cxx")
 endif()
 
-# Add abseil-cpp
-set(ABSL_PROPAGATE_CXX_STD ON)
-add_subdirectory(submodules/abseil-cpp EXCLUDE_FROM_ALL)
-
-# Add simdjson
-add_subdirectory(submodules/simdjson EXCLUDE_FROM_ALL)
-
 find_package(Threads REQUIRED)
 
-# Add yaml-cpp
-add_subdirectory(submodules/yaml-cpp EXCLUDE_FROM_ALL)
+find_package(yaml-cpp REQUIRED)
+if(yaml-cpp_FOUND)
+    message(STATUS "Found yaml-cpp ${yaml-cpp_VERSION}")
+endif()
 
-# Add ystdlib-cpp
+# Add ystdlib
 set(YSTDLIB_CPP_BUILD_TESTING OFF)
-add_subdirectory(submodules/ystdlib-cpp EXCLUDE_FROM_ALL)
+add_subdirectory("${CLP_YSTDLIB_SOURCE_DIRECTORY}" "${CMAKE_BINARY_DIR}/ystdlib" EXCLUDE_FROM_ALL)
 
 # Find and setup ZStd Library
 if(CLP_USE_STATIC_LIBS)
@@ -372,6 +411,9 @@ set(SOURCE_FILES_reducer_unitTest
     )
 
 set(SOURCE_FILES_unitTest
+        "${CLP_SQLITE3_SOURCE_DIRECTORY}/sqlite3.c"
+        "${CLP_SQLITE3_SOURCE_DIRECTORY}/sqlite3.h"
+        "${CLP_SQLITE3_SOURCE_DIRECTORY}/sqlite3ext.h"
         src/clp/aws/AwsAuthenticationSigner.cpp
         src/clp/aws/AwsAuthenticationSigner.hpp
         src/clp/aws/constants.hpp
@@ -594,9 +636,6 @@ set(SOURCE_FILES_unitTest
         src/clp/version.hpp
         src/clp/WriterInterface.cpp
         src/clp/WriterInterface.hpp
-        submodules/sqlite3/sqlite3.c
-        submodules/sqlite3/sqlite3.h
-        submodules/sqlite3/sqlite3ext.h
         tests/LogSuppressor.hpp
         tests/TestOutputCleaner.hpp
         tests/test-BoundedReader.cpp
@@ -639,30 +678,35 @@ add_executable(unitTest
         )
 target_include_directories(unitTest
         PRIVATE
+        ${CLP_OUTCOME_SOURCE_DIRECTORY}
+        ${CLP_SQLITE3_SOURCE_DIRECTORY}
         ${CMAKE_SOURCE_DIR}/submodules
         )
 target_link_libraries(unitTest
         PRIVATE
         absl::flat_hash_map
         Boost::filesystem Boost::iostreams Boost::program_options Boost::regex Boost::url
+        Catch2::Catch2
         ${CURL_LIBRARIES}
         clp_s::search::ast
         clp_s::TimestampPattern
+        date::date
         fmt::fmt
         kql
         log_surgeon::log_surgeon
         LibArchive::LibArchive
         MariaDBClient::MariaDBClient
         ${MONGOCXX_TARGET}
-        simdjson
+        nlohmann_json::nlohmann_json
+        simdjson::simdjson
         spdlog::spdlog
         sql
         OpenSSL::Crypto
         ${sqlite_LIBRARY_DEPENDENCIES}
         ${STD_FS_LIBS}
         clp::regex_utils
         clp::string_utils
-        yaml-cpp::yaml-cpp
+        yaml-cpp
         ystdlib::containers
         ystdlib::error_handling
         ${LIBLZMA_LIBRARIES}

components/core/cmake/Modules/FindANTLR.cmake

@@ -10,7 +10,6 @@ endif ()
 
 set(ANTLR4_TAG 4.13.1)
 add_definitions(-DANTLR4CPP_STATIC)
-set(ANTLR_EXECUTABLE ${PROJECT_SOURCE_DIR}/third-party/antlr/antlr-${ANTLR4_TAG}-complete.jar)
 include(ExternalAntlr4Cpp)
 
 find_package(Java 11 REQUIRED COMPONENTS Runtime)

components/core/src/clp/GlobalMetadataDBConfig.cpp

@@ -1,7 +1,7 @@
 #include "GlobalMetadataDBConfig.hpp"
 
 #include <fmt/core.h>
-#include <yaml-cpp/include/yaml-cpp/yaml.h>
+#include <yaml-cpp/yaml.h>
 
 using std::exception;
 using std::invalid_argument;

components/core/src/clp/SQLiteDB.hpp

@@ -3,8 +3,9 @@
 
 #include <string>
 
+#include <sqlite3.h>
+
 #include "ErrorCode.hpp"
-#include "sqlite3/sqlite3.h"
 #include "SQLitePreparedStatement.hpp"
 #include "TraceableException.hpp"
 

components/core/src/clp/SQLitePreparedStatement.hpp

@@ -3,7 +3,7 @@
 
 #include <string>
 
-#include <sqlite3/sqlite3.h>
+#include <sqlite3.h>
 
 #include "ErrorCode.hpp"
 #include "TraceableException.hpp"

components/core/src/clp/TimestampPattern.cpp

@@ -4,7 +4,7 @@
 #include <cstring>
 #include <vector>
 
-#include <date/include/date/date.h>
+#include <date/date.h>
 
 #include "spdlog_with_specializations.hpp"
 

components/core/src/clp/clg/CMakeLists.txt

@@ -117,28 +117,35 @@ set(
         ../version.hpp
         ../WriterInterface.cpp
         ../WriterInterface.hpp
-        "${PROJECT_SOURCE_DIR}/submodules/sqlite3/sqlite3.c"
-        "${PROJECT_SOURCE_DIR}/submodules/sqlite3/sqlite3.h"
-        "${PROJECT_SOURCE_DIR}/submodules/sqlite3/sqlite3ext.h"
+        "${CLP_SQLITE3_SOURCE_DIRECTORY}/sqlite3.c"
+        "${CLP_SQLITE3_SOURCE_DIRECTORY}/sqlite3.h"
+        "${CLP_SQLITE3_SOURCE_DIRECTORY}/sqlite3ext.h"
         clg.cpp
         CommandLineArguments.cpp
         CommandLineArguments.hpp
 )
 
 add_executable(clg ${CLG_SOURCES})
 target_compile_features(clg PRIVATE cxx_std_20)
-target_include_directories(clg PRIVATE "${PROJECT_SOURCE_DIR}/submodules")
+target_include_directories(clg
+        PRIVATE
+        "${CLP_OUTCOME_SOURCE_DIRECTORY}"
+        "${CLP_SQLITE3_SOURCE_DIRECTORY}"
+        "${PROJECT_SOURCE_DIR}/submodules"
+)
 target_link_libraries(clg
         PRIVATE
         Boost::filesystem Boost::program_options
+        date::date
         fmt::fmt
         log_surgeon::log_surgeon
         MariaDBClient::MariaDBClient
+        nlohmann_json::nlohmann_json
         spdlog::spdlog
         ${sqlite_LIBRARY_DEPENDENCIES}
         ${STD_FS_LIBS}
         clp::string_utils
-        yaml-cpp::yaml-cpp
+        yaml-cpp
         ystdlib::containers
         ZStd::ZStd
 )

components/core/src/clp/clo/CMakeLists.txt

@@ -119,9 +119,9 @@ set(
         ../version.hpp
         ../WriterInterface.cpp
         ../WriterInterface.hpp
-        "${PROJECT_SOURCE_DIR}/submodules/sqlite3/sqlite3.c"
-        "${PROJECT_SOURCE_DIR}/submodules/sqlite3/sqlite3.h"
-        "${PROJECT_SOURCE_DIR}/submodules/sqlite3/sqlite3ext.h"
+        "${CLP_SQLITE3_SOURCE_DIRECTORY}/sqlite3.c"
+        "${CLP_SQLITE3_SOURCE_DIRECTORY}/sqlite3.h"
+        "${CLP_SQLITE3_SOURCE_DIRECTORY}/sqlite3ext.h"
         clo.cpp
         CommandLineArguments.cpp
         CommandLineArguments.hpp
@@ -155,14 +155,21 @@ set(
 
 add_executable(clo ${CLO_SOURCES} ${REDUCER_SOURCES})
 target_compile_features(clo PRIVATE cxx_std_20)
-target_include_directories(clo PRIVATE "${PROJECT_SOURCE_DIR}/submodules")
+target_include_directories(clo
+        PRIVATE
+        "${CLP_OUTCOME_SOURCE_DIRECTORY}"
+        "${CLP_SQLITE3_SOURCE_DIRECTORY}"
+        "${PROJECT_SOURCE_DIR}/submodules"
+)
 target_link_libraries(clo
         PRIVATE
         Boost::filesystem Boost::program_options
+        date::date
         fmt::fmt
         log_surgeon::log_surgeon
         ${MONGOCXX_TARGET}
         msgpack-cxx
+        nlohmann_json::nlohmann_json
         spdlog::spdlog
         ${sqlite_LIBRARY_DEPENDENCIES}
         ${STD_FS_LIBS}

components/core/src/clp/clo/clo.cpp

@@ -3,8 +3,8 @@
 #include <memory>
 #include <string>
 
-#include <json/single_include/nlohmann/json.hpp>
 #include <mongocxx/instance.hpp>
+#include <nlohmann/json.hpp>
 #include <spdlog/sinks/stdout_sinks.h>
 
 #include "../../reducer/network_utils.hpp"