diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index d48d32c4f8..5d42a0f51b 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -13,7 +13,7 @@ on: - "components/core/tools/scripts/deps-download/**" - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" - "taskfile.yaml" - - "taskfiles/deps.yaml" + - "taskfiles/**" - "tools/scripts/deps-download/**" push: paths: @@ -27,7 +27,7 @@ on: - "components/core/tools/scripts/deps-download/**" - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" - "taskfile.yaml" - - "taskfiles/deps.yaml" + - "taskfiles/**" - "tools/scripts/deps-download/**" schedule: # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index f2c6424174..71c371156b 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -9,7 +9,7 @@ on: - ".gitmodules" - "components/core/**" - "taskfile.yaml" - - "taskfiles/deps.yaml" + - "taskfiles/**" - "tools/scripts/deps-download/**" - "!components/core/tools/scripts/lib_install/macos/**" push: @@ -20,7 +20,7 @@ on: - ".gitmodules" - "components/core/**" - "taskfile.yaml" - - "taskfiles/deps.yaml" + - "taskfiles/**" - "tools/scripts/deps-download/**" - "!components/core/tools/scripts/lib_install/macos/**" schedule: @@ -88,7 +88,7 @@ jobs: - "components/core/tests/**" - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" - "taskfile.yaml" - - "taskfiles/deps.yaml" + - "taskfiles/**" - "tools/scripts/deps-download/**" centos-stream-9-deps-image: diff --git a/.gitmodules b/.gitmodules index 5584aa7bb0..2d96d0d8ed 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,34 +1,3 @@ -[submodule "components/core/submodules/Catch2"] - path = components/core/submodules/Catch2 - url = https://github.com/catchorg/Catch2.git -[submodule "components/core/submodules/date"] - path = components/core/submodules/date - url = https://github.com/HowardHinnant/date.git -[submodule "components/core/submodules/json"] - path = components/core/submodules/json - url = https://github.com/nlohmann/json.git - shallow = true -[submodule "components/core/submodules/yaml-cpp"] - path = components/core/submodules/yaml-cpp - url = https://github.com/jbeder/yaml-cpp.git -[submodule "components/core/submodules/log-surgeon"] - path = components/core/submodules/log-surgeon - url = https://github.com/y-scope/log-surgeon.git -[submodule "components/core/submodules/simdjson"] - path = components/core/submodules/simdjson - url = https://github.com/simdjson/simdjson.git -[submodule "components/core/submodules/abseil-cpp"] - path = components/core/submodules/abseil-cpp - url = https://github.com/abseil/abseil-cpp.git [submodule "tools/yscope-dev-utils"] path = tools/yscope-dev-utils url = https://github.com/y-scope/yscope-dev-utils.git -[submodule "components/core/submodules/outcome"] - path = components/core/submodules/outcome - url = https://github.com/ned14/outcome.git -[submodule "components/core/submodules/utfcpp"] - path = components/core/submodules/utfcpp - url = https://github.com/nemtrif/utfcpp.git -[submodule "components/core/submodules/ystdlib-cpp"] - path = components/core/submodules/ystdlib-cpp - url = https://github.com/y-scope/ystdlib-cpp.git diff --git a/components/core/.clang-format b/components/core/.clang-format index bcd2660465..626220791f 100644 --- a/components/core/.clang-format +++ b/components/core/.clang-format @@ -4,9 +4,9 @@ IncludeCategories: # NOTE: A header is grouped by first matching regex # Library headers. Update when adding new libraries. # NOTE: clang-format retains leading white-space on a line in violation of the YAML spec. - - Regex: "<(absl|antlr4|archive|boost|bsoncxx|catch2|curl|date|fmt|json|log_surgeon|lzma|mongocxx\ -|msgpack|mysql|openssl|outcome|regex_utils|simdjson|spdlog|sqlite3|string_utils|yaml-cpp|ystdlib\ -|zstd)" + - Regex: "<(absl|antlr4|archive|boost|bsoncxx|catch2|curl|date|fmt|log_surgeon|lzma|mongocxx\ +|msgpack|mysql|nlohmann|openssl|outcome|regex_utils|simdjson|spdlog|sqlite3|string_utils|yaml-cpp\ +|ystdlib|zstd)" Priority: 3 # C system headers - Regex: "^<.+\\.h>" diff --git a/components/core/.gitignore b/components/core/.gitignore index 18670ec68b..f65519e3cc 100644 --- a/components/core/.gitignore +++ b/components/core/.gitignore @@ -1,3 +1 @@ build/** -submodules/sqlite3/* -third-party/** diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index e0da1932c1..320475355a 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -96,6 +96,28 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") endif () endif () +if(PROJECT_IS_TOP_LEVEL) + # Include dependency settings if the project isn't being included as a subproject. + # NOTE: We mark the file optional since it's not required if the user happens to have the + # dependencies installed already. + include("${CMAKE_SOURCE_DIR}/../../build/deps/core/cmake-settings/all.cmake" + OPTIONAL + RESULT_VARIABLE CLP_DEPS_SETTINGS_FILE_PATH + ) + + if(NOT CLP_DEPS_SETTINGS_FILE_PATH STREQUAL "NOTFOUND") + # Set CMP0144 since our minimum required CMake version is less than 3.27. + if(POLICY "CMP0144") + cmake_policy(SET "CMP0144" "NEW") + endif() + endif() +endif() + +find_package(absl REQUIRED) +if (absl_FOUND) + message(STATUS "Found absl ${absl_VERSION}") +endif() + # Find and setup ANTLR Library # We build and link to the static library find_package(ANTLR REQUIRED) @@ -116,6 +138,16 @@ else() message(FATAL_ERROR "Could not find ${CLP_LIBS_STRING} libraries for Boost") endif() +find_package(Catch2 REQUIRED) +if (Catch2_FOUND) + message(STATUS "Found Catch2 ${Catch2_VERSION}") +endif() + +find_package(date REQUIRED) +if (date_FOUND) + message(STATUS "Found date ${date_VERSION}") +endif() + # Find and setup fmt # NOTE: # - We only try to link to the static library @@ -127,6 +159,21 @@ else() message(FATAL_ERROR "Could not find static libraries for fmt") endif() +find_package(log_surgeon REQUIRED) +if(log_surgeon_FOUND) + message(STATUS "Found log_surgeon ${log_surgeon_VERSION}") +endif() + +find_package(nlohmann_json REQUIRED) +if(nlohmann_json_FOUND) + message(STATUS "Found nlohmann_json ${nlohmann_json_VERSION}") +endif() + +find_package(simdjson REQUIRED) +if(simdjson_FOUND) + message(STATUS "Found simdjson ${simdjson_VERSION}") +endif() + # Find and setup spdlog if(CLP_USE_STATIC_LIBS) # NOTE: On some Linux distributions (e.g. Ubuntu), the spdlog package only contains a dynamic @@ -175,9 +222,6 @@ else () message(FATAL_ERROR "OpenSSL not found") endif () -# Add log surgeon -add_subdirectory(submodules/log-surgeon EXCLUDE_FROM_ALL) - # Find and setup MariaDBClient library if(CLP_USE_STATIC_LIBS) # NOTE: We can't statically link to MariaDBClient since it's GPL @@ -212,21 +256,16 @@ else() message(FATAL_ERROR "Could not find msgpack-cxx") endif() -# Add abseil-cpp -set(ABSL_PROPAGATE_CXX_STD ON) -add_subdirectory(submodules/abseil-cpp EXCLUDE_FROM_ALL) - -# Add simdjson -add_subdirectory(submodules/simdjson EXCLUDE_FROM_ALL) - find_package(Threads REQUIRED) -# Add yaml-cpp -add_subdirectory(submodules/yaml-cpp EXCLUDE_FROM_ALL) +find_package(yaml-cpp REQUIRED) +if(yaml-cpp_FOUND) + message(STATUS "Found yaml-cpp ${yaml-cpp_VERSION}") +endif() -# Add ystdlib-cpp +# Add ystdlib set(YSTDLIB_CPP_BUILD_TESTING OFF) -add_subdirectory(submodules/ystdlib-cpp EXCLUDE_FROM_ALL) +add_subdirectory("${CLP_YSTDLIB_SOURCE_DIRECTORY}" "${CMAKE_BINARY_DIR}/ystdlib" EXCLUDE_FROM_ALL) # Find and setup ZStd Library if(CLP_USE_STATIC_LIBS) @@ -372,6 +411,7 @@ set(SOURCE_FILES_reducer_unitTest ) set(SOURCE_FILES_unitTest + "${CLP_SQLITE3_SOURCE_DIRECTORY}/sqlite3.c" src/clp/aws/AwsAuthenticationSigner.cpp src/clp/aws/AwsAuthenticationSigner.hpp src/clp/aws/constants.hpp @@ -595,9 +635,6 @@ set(SOURCE_FILES_unitTest src/clp/version.hpp src/clp/WriterInterface.cpp src/clp/WriterInterface.hpp - submodules/sqlite3/sqlite3.c - submodules/sqlite3/sqlite3.h - submodules/sqlite3/sqlite3ext.h tests/LogSuppressor.hpp tests/TestOutputCleaner.hpp tests/test-BoundedReader.cpp @@ -640,22 +677,26 @@ add_executable(unitTest ) target_include_directories(unitTest PRIVATE - ${CMAKE_SOURCE_DIR}/submodules + ${CLP_OUTCOME_INCLUDE_DIRECTORY} + ${CLP_SQLITE3_INCLUDE_DIRECTORY} ) target_link_libraries(unitTest PRIVATE absl::flat_hash_map Boost::filesystem Boost::iostreams Boost::program_options Boost::regex Boost::url + Catch2::Catch2 ${CURL_LIBRARIES} clp_s::search::ast clp_s::TimestampPattern + date::date fmt::fmt kql log_surgeon::log_surgeon LibArchive::LibArchive MariaDBClient::MariaDBClient ${MONGOCXX_TARGET} - simdjson + nlohmann_json::nlohmann_json + simdjson::simdjson spdlog::spdlog sql OpenSSL::Crypto @@ -663,7 +704,7 @@ target_link_libraries(unitTest ${STD_FS_LIBS} clp::regex_utils clp::string_utils - yaml-cpp::yaml-cpp + yaml-cpp ystdlib::containers ystdlib::error_handling ${LIBLZMA_LIBRARIES} diff --git a/components/core/cmake/Modules/FindANTLR.cmake b/components/core/cmake/Modules/FindANTLR.cmake index d191ba071c..6e2f96d2b5 100644 --- a/components/core/cmake/Modules/FindANTLR.cmake +++ b/components/core/cmake/Modules/FindANTLR.cmake @@ -10,7 +10,6 @@ endif () set(ANTLR4_TAG 4.13.1) add_definitions(-DANTLR4CPP_STATIC) -set(ANTLR_EXECUTABLE ${PROJECT_SOURCE_DIR}/third-party/antlr/antlr-${ANTLR4_TAG}-complete.jar) include(ExternalAntlr4Cpp) find_package(Java 11 REQUIRED COMPONENTS Runtime) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index dcebece9ca..833c72d6df 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -1,7 +1,7 @@ #include "GlobalMetadataDBConfig.hpp" #include -#include +#include using std::exception; using std::invalid_argument; diff --git a/components/core/src/clp/SQLiteDB.hpp b/components/core/src/clp/SQLiteDB.hpp index 74c5ace17a..01b993dd7e 100644 --- a/components/core/src/clp/SQLiteDB.hpp +++ b/components/core/src/clp/SQLiteDB.hpp @@ -3,8 +3,9 @@ #include +#include + #include "ErrorCode.hpp" -#include "sqlite3/sqlite3.h" #include "SQLitePreparedStatement.hpp" #include "TraceableException.hpp" diff --git a/components/core/src/clp/TimestampPattern.cpp b/components/core/src/clp/TimestampPattern.cpp index c7a8bf78fb..566aa8c92a 100644 --- a/components/core/src/clp/TimestampPattern.cpp +++ b/components/core/src/clp/TimestampPattern.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include #include "spdlog_with_specializations.hpp" diff --git a/components/core/src/clp/clg/CMakeLists.txt b/components/core/src/clp/clg/CMakeLists.txt index 0ec86fc7ed..c8a06ec94d 100644 --- a/components/core/src/clp/clg/CMakeLists.txt +++ b/components/core/src/clp/clg/CMakeLists.txt @@ -117,9 +117,7 @@ set( ../version.hpp ../WriterInterface.cpp ../WriterInterface.hpp - "${PROJECT_SOURCE_DIR}/submodules/sqlite3/sqlite3.c" - "${PROJECT_SOURCE_DIR}/submodules/sqlite3/sqlite3.h" - "${PROJECT_SOURCE_DIR}/submodules/sqlite3/sqlite3ext.h" + "${CLP_SQLITE3_SOURCE_DIRECTORY}/sqlite3.c" clg.cpp CommandLineArguments.cpp CommandLineArguments.hpp @@ -127,18 +125,24 @@ set( add_executable(clg ${CLG_SOURCES}) target_compile_features(clg PRIVATE cxx_std_20) -target_include_directories(clg PRIVATE "${PROJECT_SOURCE_DIR}/submodules") +target_include_directories(clg + PRIVATE + "${CLP_OUTCOME_INCLUDE_DIRECTORY}" + "${CLP_SQLITE3_INCLUDE_DIRECTORY}" +) target_link_libraries(clg PRIVATE Boost::filesystem Boost::program_options + date::date fmt::fmt log_surgeon::log_surgeon MariaDBClient::MariaDBClient + nlohmann_json::nlohmann_json spdlog::spdlog ${sqlite_LIBRARY_DEPENDENCIES} ${STD_FS_LIBS} clp::string_utils - yaml-cpp::yaml-cpp + yaml-cpp ystdlib::containers ZStd::ZStd ) diff --git a/components/core/src/clp/clo/CMakeLists.txt b/components/core/src/clp/clo/CMakeLists.txt index f0cc80037a..a122b7ea01 100644 --- a/components/core/src/clp/clo/CMakeLists.txt +++ b/components/core/src/clp/clo/CMakeLists.txt @@ -119,9 +119,7 @@ set( ../version.hpp ../WriterInterface.cpp ../WriterInterface.hpp - "${PROJECT_SOURCE_DIR}/submodules/sqlite3/sqlite3.c" - "${PROJECT_SOURCE_DIR}/submodules/sqlite3/sqlite3.h" - "${PROJECT_SOURCE_DIR}/submodules/sqlite3/sqlite3ext.h" + "${CLP_SQLITE3_SOURCE_DIRECTORY}/sqlite3.c" clo.cpp CommandLineArguments.cpp CommandLineArguments.hpp @@ -155,14 +153,20 @@ set( add_executable(clo ${CLO_SOURCES} ${REDUCER_SOURCES}) target_compile_features(clo PRIVATE cxx_std_20) -target_include_directories(clo PRIVATE "${PROJECT_SOURCE_DIR}/submodules") +target_include_directories(clo + PRIVATE + "${CLP_OUTCOME_INCLUDE_DIRECTORY}" + "${CLP_SQLITE3_INCLUDE_DIRECTORY}" +) target_link_libraries(clo PRIVATE Boost::filesystem Boost::program_options + date::date fmt::fmt log_surgeon::log_surgeon ${MONGOCXX_TARGET} msgpack-cxx + nlohmann_json::nlohmann_json spdlog::spdlog ${sqlite_LIBRARY_DEPENDENCIES} ${STD_FS_LIBS} diff --git a/components/core/src/clp/clo/clo.cpp b/components/core/src/clp/clo/clo.cpp index 3412e2d079..a675f9fe6a 100644 --- a/components/core/src/clp/clo/clo.cpp +++ b/components/core/src/clp/clo/clo.cpp @@ -3,8 +3,8 @@ #include #include -#include #include +#include #include #include "../../reducer/network_utils.hpp" diff --git a/components/core/src/clp/clp/CMakeLists.txt b/components/core/src/clp/clp/CMakeLists.txt index c0ecaf4636..d990ea1a74 100644 --- a/components/core/src/clp/clp/CMakeLists.txt +++ b/components/core/src/clp/clp/CMakeLists.txt @@ -147,8 +147,7 @@ set( ../version.hpp ../WriterInterface.cpp ../WriterInterface.hpp - "${PROJECT_SOURCE_DIR}/submodules/sqlite3/sqlite3.c" - "${PROJECT_SOURCE_DIR}/submodules/sqlite3/sqlite3.h" + "${CLP_SQLITE3_SOURCE_DIRECTORY}/sqlite3.c" clp.cpp CommandLineArguments.cpp CommandLineArguments.hpp @@ -168,19 +167,25 @@ set( add_executable(clp ${CLP_SOURCES}) target_compile_features(clp PRIVATE cxx_std_20) -target_include_directories(clp PRIVATE "${PROJECT_SOURCE_DIR}/submodules") +target_include_directories(clp + PRIVATE + "${CLP_OUTCOME_INCLUDE_DIRECTORY}" + "${CLP_SQLITE3_INCLUDE_DIRECTORY}" +) target_link_libraries(clp PRIVATE Boost::filesystem Boost::program_options + date::date fmt::fmt log_surgeon::log_surgeon spdlog::spdlog ${sqlite_LIBRARY_DEPENDENCIES} LibArchive::LibArchive MariaDBClient::MariaDBClient + nlohmann_json::nlohmann_json ${STD_FS_LIBS} clp::string_utils - yaml-cpp::yaml-cpp + yaml-cpp ystdlib::containers ZStd::ZStd ) diff --git a/components/core/src/clp/ffi/KeyValuePairLogEvent.cpp b/components/core/src/clp/ffi/KeyValuePairLogEvent.cpp index 3918c01843..c65f2984c9 100644 --- a/components/core/src/clp/ffi/KeyValuePairLogEvent.cpp +++ b/components/core/src/clp/ffi/KeyValuePairLogEvent.cpp @@ -11,8 +11,9 @@ #include #include -#include -#include +#include +#include +#include #include "../ir/EncodedTextAst.hpp" #include "../time_types.hpp" diff --git a/components/core/src/clp/ffi/KeyValuePairLogEvent.hpp b/components/core/src/clp/ffi/KeyValuePairLogEvent.hpp index 02aa058432..de5b6b71f1 100644 --- a/components/core/src/clp/ffi/KeyValuePairLogEvent.hpp +++ b/components/core/src/clp/ffi/KeyValuePairLogEvent.hpp @@ -7,8 +7,8 @@ #include #include -#include -#include +#include +#include #include "../time_types.hpp" #include "SchemaTree.hpp" diff --git a/components/core/src/clp/ffi/ir_stream/Deserializer.hpp b/components/core/src/clp/ffi/ir_stream/Deserializer.hpp index 53e762fdc6..5aecee39c8 100644 --- a/components/core/src/clp/ffi/ir_stream/Deserializer.hpp +++ b/components/core/src/clp/ffi/ir_stream/Deserializer.hpp @@ -9,8 +9,8 @@ #include #include -#include -#include +#include +#include #include "../../ReaderInterface.hpp" #include "../../time_types.hpp" diff --git a/components/core/src/clp/ffi/ir_stream/Serializer.cpp b/components/core/src/clp/ffi/ir_stream/Serializer.cpp index 34ea032887..0928b7485f 100644 --- a/components/core/src/clp/ffi/ir_stream/Serializer.cpp +++ b/components/core/src/clp/ffi/ir_stream/Serializer.cpp @@ -11,9 +11,9 @@ #include #include -#include #include -#include +#include +#include #include "../../ir/types.hpp" #include "../../time_types.hpp" diff --git a/components/core/src/clp/ffi/ir_stream/Serializer.hpp b/components/core/src/clp/ffi/ir_stream/Serializer.hpp index 3f4bc2f1b4..6ee93280e5 100644 --- a/components/core/src/clp/ffi/ir_stream/Serializer.hpp +++ b/components/core/src/clp/ffi/ir_stream/Serializer.hpp @@ -7,9 +7,9 @@ #include #include -#include #include -#include +#include +#include #include "../../time_types.hpp" #include "../SchemaTree.hpp" diff --git a/components/core/src/clp/ffi/ir_stream/encoding_methods.cpp b/components/core/src/clp/ffi/ir_stream/encoding_methods.cpp index adec027079..dec8dca97f 100644 --- a/components/core/src/clp/ffi/ir_stream/encoding_methods.cpp +++ b/components/core/src/clp/ffi/ir_stream/encoding_methods.cpp @@ -1,6 +1,6 @@ #include "encoding_methods.hpp" -#include +#include #include "../../ir/parsing.hpp" #include "../../ir/types.hpp" diff --git a/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.cpp b/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.cpp index df8e3aaf22..6fc6a4e54d 100644 --- a/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.cpp +++ b/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.cpp @@ -10,7 +10,7 @@ #include #include -#include +#include #include "../../ErrorCode.hpp" #include "../../ir/EncodedTextAst.hpp" diff --git a/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.hpp b/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.hpp index 3857caf9b9..cbd1563a78 100644 --- a/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.hpp +++ b/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.hpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include "../../ReaderInterface.hpp" #include "../../time_types.hpp" diff --git a/components/core/src/clp/ffi/ir_stream/search/NewProjectedSchemaTreeNodeCallbackReq.hpp b/components/core/src/clp/ffi/ir_stream/search/NewProjectedSchemaTreeNodeCallbackReq.hpp index 1bebcd30d6..4c114def3a 100644 --- a/components/core/src/clp/ffi/ir_stream/search/NewProjectedSchemaTreeNodeCallbackReq.hpp +++ b/components/core/src/clp/ffi/ir_stream/search/NewProjectedSchemaTreeNodeCallbackReq.hpp @@ -4,7 +4,7 @@ #include #include -#include +#include #include "../../SchemaTree.hpp" diff --git a/components/core/src/clp/ffi/ir_stream/utils.cpp b/components/core/src/clp/ffi/ir_stream/utils.cpp index 371d7cfba2..ab6040c2d0 100644 --- a/components/core/src/clp/ffi/ir_stream/utils.cpp +++ b/components/core/src/clp/ffi/ir_stream/utils.cpp @@ -5,7 +5,8 @@ #include #include -#include +#include +#include #include "../../type_utils.hpp" #include "decoding_methods.hpp" diff --git a/components/core/src/clp/ffi/ir_stream/utils.hpp b/components/core/src/clp/ffi/ir_stream/utils.hpp index fc66a59296..654416a0f7 100644 --- a/components/core/src/clp/ffi/ir_stream/utils.hpp +++ b/components/core/src/clp/ffi/ir_stream/utils.hpp @@ -9,8 +9,8 @@ #include #include -#include -#include +#include +#include #include "../../ErrorCode.hpp" #include "../../ir/types.hpp" diff --git a/components/core/src/clp/ir/LogEventDeserializer.cpp b/components/core/src/clp/ir/LogEventDeserializer.cpp index a2bfc87db4..09622dcb8f 100644 --- a/components/core/src/clp/ir/LogEventDeserializer.cpp +++ b/components/core/src/clp/ir/LogEventDeserializer.cpp @@ -2,8 +2,8 @@ #include -#include -#include +#include +#include #include #include "../ffi/ir_stream/decoding_methods.hpp" diff --git a/components/core/src/clp/ir/LogEventDeserializer.hpp b/components/core/src/clp/ir/LogEventDeserializer.hpp index 6a6bc049a9..8c8052b807 100644 --- a/components/core/src/clp/ir/LogEventDeserializer.hpp +++ b/components/core/src/clp/ir/LogEventDeserializer.hpp @@ -3,7 +3,7 @@ #include -#include +#include #include "../ReaderInterface.hpp" #include "../time_types.hpp" diff --git a/components/core/src/clp/make_dictionaries_readable/CMakeLists.txt b/components/core/src/clp/make_dictionaries_readable/CMakeLists.txt index 8834aad2b6..01154bba7d 100644 --- a/components/core/src/clp/make_dictionaries_readable/CMakeLists.txt +++ b/components/core/src/clp/make_dictionaries_readable/CMakeLists.txt @@ -34,7 +34,6 @@ set( ../VariableDictionaryReader.hpp ../WriterInterface.cpp ../WriterInterface.hpp - "${PROJECT_SOURCE_DIR}/submodules/date/include/date/date.h" CommandLineArguments.cpp CommandLineArguments.hpp make-dictionaries-readable.cpp @@ -42,7 +41,6 @@ set( add_executable(make-dictionaries-readable ${MAKE_DICTIONARIES_READABLE_SOURCES}) target_compile_features(make-dictionaries-readable PRIVATE cxx_std_20) -target_include_directories(make-dictionaries-readable PRIVATE "${PROJECT_SOURCE_DIR}/submodules") target_link_libraries(make-dictionaries-readable PRIVATE Boost::filesystem Boost::program_options diff --git a/components/core/src/clp/regex_utils/CMakeLists.txt b/components/core/src/clp/regex_utils/CMakeLists.txt index 813ac938c2..a359702833 100644 --- a/components/core/src/clp/regex_utils/CMakeLists.txt +++ b/components/core/src/clp/regex_utils/CMakeLists.txt @@ -15,7 +15,7 @@ add_library(clp::regex_utils ALIAS regex_utils) target_include_directories(regex_utils PUBLIC ../ - "${PROJECT_SOURCE_DIR}/submodules" + "${CLP_OUTCOME_INCLUDE_DIRECTORY}" ) target_link_libraries(regex_utils PUBLIC diff --git a/components/core/src/clp/regex_utils/regex_translation_utils.cpp b/components/core/src/clp/regex_utils/regex_translation_utils.cpp index 51897b8d91..d3f2882313 100644 --- a/components/core/src/clp/regex_utils/regex_translation_utils.cpp +++ b/components/core/src/clp/regex_utils/regex_translation_utils.cpp @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include "constants.hpp" diff --git a/components/core/src/clp/regex_utils/regex_translation_utils.hpp b/components/core/src/clp/regex_utils/regex_translation_utils.hpp index 9b3a8f921a..0489c68e49 100644 --- a/components/core/src/clp/regex_utils/regex_translation_utils.hpp +++ b/components/core/src/clp/regex_utils/regex_translation_utils.hpp @@ -4,7 +4,7 @@ #include #include -#include +#include #include "regex_utils/RegexToWildcardTranslatorConfig.hpp" diff --git a/components/core/src/clp/streaming_archive/writer/Archive.cpp b/components/core/src/clp/streaming_archive/writer/Archive.cpp index 688cf3c23e..a4d09601a1 100644 --- a/components/core/src/clp/streaming_archive/writer/Archive.cpp +++ b/components/core/src/clp/streaming_archive/writer/Archive.cpp @@ -10,9 +10,9 @@ #include #include #include -#include #include #include +#include #include "../../EncodedVariableInterpreter.hpp" #include "../../ir/types.hpp" diff --git a/components/core/src/clp_s/ArchiveWriter.cpp b/components/core/src/clp_s/ArchiveWriter.cpp index a05856da16..58c52d3381 100644 --- a/components/core/src/clp_s/ArchiveWriter.cpp +++ b/components/core/src/clp_s/ArchiveWriter.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include #include "../clp/streaming_archive/Constants.hpp" #include "archive_constants.hpp" diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 171a681dcd..de5273106d 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -73,7 +73,6 @@ set( set( CLP_S_SOURCES - "${PROJECT_SOURCE_DIR}/submodules/date/include/date/date.h" archive_constants.hpp ArchiveReader.cpp ArchiveReader.hpp @@ -205,12 +204,12 @@ add_library( ${CLP_S_TIMESTAMP_PATTERN_SOURCES} ) add_library(clp_s::TimestampPattern ALIAS clp_s_TimestampPattern) -target_include_directories(clp_s_TimestampPattern PRIVATE "${PROJECT_SOURCE_DIR}/submodules") target_compile_features(clp_s_TimestampPattern PRIVATE cxx_std_20) target_link_libraries( clp_s_TimestampPattern PRIVATE clp::string_utils + date::date ) add_executable( @@ -235,14 +234,18 @@ target_link_libraries( MariaDBClient::MariaDBClient ${MONGOCXX_TARGET} msgpack-cxx + nlohmann_json::nlohmann_json OpenSSL::Crypto - simdjson + simdjson::simdjson spdlog::spdlog - yaml-cpp::yaml-cpp + yaml-cpp ystdlib::containers ZStd::ZStd ) -target_include_directories(clp-s PRIVATE "${PROJECT_SOURCE_DIR}/submodules") +target_include_directories(clp-s + PRIVATE + "${CLP_OUTCOME_INCLUDE_DIRECTORY}" +) set_target_properties( clp-s PROPERTIES diff --git a/components/core/src/clp_s/JsonConstructor.cpp b/components/core/src/clp_s/JsonConstructor.cpp index 515d974043..b4995f4cd4 100644 --- a/components/core/src/clp_s/JsonConstructor.cpp +++ b/components/core/src/clp_s/JsonConstructor.cpp @@ -5,11 +5,11 @@ #include #include -#include #include #include #include #include +#include #include "archive_constants.hpp" #include "ErrorCode.hpp" diff --git a/components/core/src/clp_s/TimestampPattern.cpp b/components/core/src/clp_s/TimestampPattern.cpp index 25131db7e4..f5500e3aad 100644 --- a/components/core/src/clp_s/TimestampPattern.cpp +++ b/components/core/src/clp_s/TimestampPattern.cpp @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 0069bdc057..b559e058e8 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -7,8 +7,8 @@ #include #include -#include #include +#include #include #include diff --git a/components/core/src/clp_s/indexer/CMakeLists.txt b/components/core/src/clp_s/indexer/CMakeLists.txt index 0dcc134ff0..98fefeaff7 100644 --- a/components/core/src/clp_s/indexer/CMakeLists.txt +++ b/components/core/src/clp_s/indexer/CMakeLists.txt @@ -82,18 +82,18 @@ set( add_executable(indexer ${INDEXER_SOURCES}) target_compile_features(indexer PRIVATE cxx_std_20) -target_include_directories(indexer PRIVATE "${PROJECT_SOURCE_DIR}/submodules") target_link_libraries(indexer PRIVATE absl::flat_hash_map Boost::iostreams Boost::program_options Boost::url ${CURL_LIBRARIES} clp::string_utils + date::date MariaDBClient::MariaDBClient OpenSSL::Crypto - simdjson + simdjson::simdjson spdlog::spdlog - yaml-cpp::yaml-cpp + yaml-cpp ystdlib::containers ZStd::ZStd ) diff --git a/components/core/src/clp_s/search/ast/CMakeLists.txt b/components/core/src/clp_s/search/ast/CMakeLists.txt index 17cda32a44..e46455f998 100644 --- a/components/core/src/clp_s/search/ast/CMakeLists.txt +++ b/components/core/src/clp_s/search/ast/CMakeLists.txt @@ -55,12 +55,11 @@ add_library( ${CLP_S_CORE_SOURCES} ) add_library(clp_s::search::ast ALIAS clp_s_search_ast) -target_include_directories(clp_s_search_ast PRIVATE "${PROJECT_SOURCE_DIR}/submodules") target_compile_features(clp_s_search_ast PRIVATE cxx_std_20) target_link_libraries( clp_s_search_ast PUBLIC clp_s::TimestampPattern PRIVATE - simdjson + simdjson::simdjson ) diff --git a/components/core/src/glt/GlobalMetadataDBConfig.cpp b/components/core/src/glt/GlobalMetadataDBConfig.cpp index d8de7c25df..460b36b239 100644 --- a/components/core/src/glt/GlobalMetadataDBConfig.cpp +++ b/components/core/src/glt/GlobalMetadataDBConfig.cpp @@ -1,7 +1,7 @@ #include "GlobalMetadataDBConfig.hpp" #include -#include +#include using std::exception; using std::invalid_argument; diff --git a/components/core/src/glt/SQLiteDB.hpp b/components/core/src/glt/SQLiteDB.hpp index ea868d42b5..668bc5b486 100644 --- a/components/core/src/glt/SQLiteDB.hpp +++ b/components/core/src/glt/SQLiteDB.hpp @@ -3,8 +3,9 @@ #include +#include + #include "ErrorCode.hpp" -#include "sqlite3/sqlite3.h" #include "SQLitePreparedStatement.hpp" #include "TraceableException.hpp" diff --git a/components/core/src/glt/TimestampPattern.cpp b/components/core/src/glt/TimestampPattern.cpp index d725906e39..7f42d93e4f 100644 --- a/components/core/src/glt/TimestampPattern.cpp +++ b/components/core/src/glt/TimestampPattern.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include #include "spdlog_with_specializations.hpp" diff --git a/components/core/src/glt/ffi/ir_stream/encoding_methods.cpp b/components/core/src/glt/ffi/ir_stream/encoding_methods.cpp index f6f352a78c..dac27fa47d 100644 --- a/components/core/src/glt/ffi/ir_stream/encoding_methods.cpp +++ b/components/core/src/glt/ffi/ir_stream/encoding_methods.cpp @@ -1,6 +1,6 @@ #include "encoding_methods.hpp" -#include +#include #include "../../ir/parsing.hpp" #include "../../ir/types.hpp" diff --git a/components/core/src/glt/glt/CMakeLists.txt b/components/core/src/glt/glt/CMakeLists.txt index 0c7a6af4a7..15062e6676 100644 --- a/components/core/src/glt/glt/CMakeLists.txt +++ b/components/core/src/glt/glt/CMakeLists.txt @@ -151,8 +151,7 @@ set( ../version.hpp ../WriterInterface.cpp ../WriterInterface.hpp - "${PROJECT_SOURCE_DIR}/submodules/sqlite3/sqlite3.c" - "${PROJECT_SOURCE_DIR}/submodules/sqlite3/sqlite3.h" + "${CLP_SQLITE3_SOURCE_DIRECTORY}/sqlite3.c" glt.cpp CommandLineArguments.cpp CommandLineArguments.hpp @@ -174,18 +173,24 @@ set( add_executable(glt ${GLT_SOURCES}) target_compile_features(glt PRIVATE cxx_std_20) -target_include_directories(glt PRIVATE "${PROJECT_SOURCE_DIR}/submodules") +target_include_directories(glt + PRIVATE + "${CLP_OUTCOME_INCLUDE_DIRECTORY}" + "${CLP_SQLITE3_INCLUDE_DIRECTORY}" +) target_link_libraries(glt PRIVATE Boost::filesystem Boost::iostreams Boost::program_options + date::date fmt::fmt spdlog::spdlog ${sqlite_LIBRARY_DEPENDENCIES} LibArchive::LibArchive MariaDBClient::MariaDBClient + nlohmann_json::nlohmann_json ${STD_FS_LIBS} clp::string_utils - yaml-cpp::yaml-cpp + yaml-cpp ZStd::ZStd ) # Put the built executable at the root of the build directory diff --git a/components/core/src/glt/ir/LogEventDeserializer.cpp b/components/core/src/glt/ir/LogEventDeserializer.cpp index fa387421a5..9b972c46a6 100644 --- a/components/core/src/glt/ir/LogEventDeserializer.cpp +++ b/components/core/src/glt/ir/LogEventDeserializer.cpp @@ -2,8 +2,8 @@ #include -#include -#include +#include +#include #include #include "../ffi/ir_stream/decoding_methods.hpp" diff --git a/components/core/src/glt/ir/LogEventDeserializer.hpp b/components/core/src/glt/ir/LogEventDeserializer.hpp index 092b0f58b2..e9d32e4083 100644 --- a/components/core/src/glt/ir/LogEventDeserializer.hpp +++ b/components/core/src/glt/ir/LogEventDeserializer.hpp @@ -3,7 +3,7 @@ #include -#include +#include #include "../ReaderInterface.hpp" #include "../TimestampPattern.hpp" diff --git a/components/core/src/glt/streaming_archive/writer/Archive.cpp b/components/core/src/glt/streaming_archive/writer/Archive.cpp index cbe7e2dbfe..dc99ec19e9 100644 --- a/components/core/src/glt/streaming_archive/writer/Archive.cpp +++ b/components/core/src/glt/streaming_archive/writer/Archive.cpp @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include "../../EncodedVariableInterpreter.hpp" #include "../../ir/types.hpp" diff --git a/components/core/src/reducer/CMakeLists.txt b/components/core/src/reducer/CMakeLists.txt index b0017a583f..72b4cf1a24 100644 --- a/components/core/src/reducer/CMakeLists.txt +++ b/components/core/src/reducer/CMakeLists.txt @@ -40,7 +40,6 @@ set( add_executable(reducer-server ${REDUCER_SOURCES}) target_compile_features(reducer-server PRIVATE cxx_std_20) -target_include_directories(reducer-server PRIVATE "${PROJECT_SOURCE_DIR}/submodules") target_link_libraries(reducer-server PRIVATE Boost::program_options @@ -49,6 +48,7 @@ target_link_libraries(reducer-server fmt::fmt ${MONGOCXX_TARGET} msgpack-cxx + nlohmann_json::nlohmann_json spdlog::spdlog ) # Put the built executable at the root of the build directory diff --git a/components/core/src/reducer/DeserializedRecordGroup.cpp b/components/core/src/reducer/DeserializedRecordGroup.cpp index 9c78086b60..a7c5b8a8f6 100644 --- a/components/core/src/reducer/DeserializedRecordGroup.cpp +++ b/components/core/src/reducer/DeserializedRecordGroup.cpp @@ -2,7 +2,7 @@ #include -#include +#include namespace reducer { DeserializedRecordGroup::DeserializedRecordGroup(std::vector& serialized_data) diff --git a/components/core/src/reducer/DeserializedRecordGroup.hpp b/components/core/src/reducer/DeserializedRecordGroup.hpp index e016ddc386..146d4907a4 100644 --- a/components/core/src/reducer/DeserializedRecordGroup.hpp +++ b/components/core/src/reducer/DeserializedRecordGroup.hpp @@ -5,7 +5,7 @@ #include #include -#include +#include #include "ConstRecordIterator.hpp" #include "JsonArrayRecordIterator.hpp" diff --git a/components/core/src/reducer/JsonArrayRecordIterator.hpp b/components/core/src/reducer/JsonArrayRecordIterator.hpp index ae46ca2c02..57f5e6d63d 100644 --- a/components/core/src/reducer/JsonArrayRecordIterator.hpp +++ b/components/core/src/reducer/JsonArrayRecordIterator.hpp @@ -1,7 +1,7 @@ #ifndef REDUCER_JSONARRAYRECORDITERATOR_HPP #define REDUCER_JSONARRAYRECORDITERATOR_HPP -#include +#include #include "ConstRecordIterator.hpp" #include "JsonRecord.hpp" diff --git a/components/core/src/reducer/JsonRecord.hpp b/components/core/src/reducer/JsonRecord.hpp index 4d8df393ae..a848cf293c 100644 --- a/components/core/src/reducer/JsonRecord.hpp +++ b/components/core/src/reducer/JsonRecord.hpp @@ -1,7 +1,7 @@ #ifndef REDUCER_JSONRECORD_HPP #define REDUCER_JSONRECORD_HPP -#include +#include #include "Record.hpp" diff --git a/components/core/src/reducer/ServerContext.cpp b/components/core/src/reducer/ServerContext.cpp index 75b667da43..773951b730 100644 --- a/components/core/src/reducer/ServerContext.cpp +++ b/components/core/src/reducer/ServerContext.cpp @@ -1,7 +1,6 @@ #include "ServerContext.hpp" #include -#include #include #include #include @@ -10,6 +9,7 @@ #include #include #include +#include #include "../clp/spdlog_with_specializations.hpp" #include "CommandLineArguments.hpp" diff --git a/components/core/src/reducer/ServerContext.hpp b/components/core/src/reducer/ServerContext.hpp index 02a0467d86..47433258b0 100644 --- a/components/core/src/reducer/ServerContext.hpp +++ b/components/core/src/reducer/ServerContext.hpp @@ -5,9 +5,9 @@ #include #include -#include #include #include +#include #include "../clp/TraceableException.hpp" #include "CommandLineArguments.hpp" diff --git a/components/core/src/reducer/reducer_server.cpp b/components/core/src/reducer/reducer_server.cpp index b5f36be778..37409f2c9a 100644 --- a/components/core/src/reducer/reducer_server.cpp +++ b/components/core/src/reducer/reducer_server.cpp @@ -4,9 +4,9 @@ #include #include -#include #include #include +#include #include #include "../clp/spdlog_with_specializations.hpp" diff --git a/components/core/submodules/Catch2 b/components/core/submodules/Catch2 deleted file mode 160000 index c4e3767e26..0000000000 --- a/components/core/submodules/Catch2 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c4e3767e265808590986d5db6ca1b5532a7f3d13 diff --git a/components/core/submodules/abseil-cpp b/components/core/submodules/abseil-cpp deleted file mode 160000 index fb3621f4f8..0000000000 --- a/components/core/submodules/abseil-cpp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fb3621f4f897824c0dbe0615fa94543df6192f30 diff --git a/components/core/submodules/date b/components/core/submodules/date deleted file mode 160000 index 6e921e1b1d..0000000000 --- a/components/core/submodules/date +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6e921e1b1d21e84a5c82416ba7ecd98e33a436d0 diff --git a/components/core/submodules/json b/components/core/submodules/json deleted file mode 160000 index 9cca280a4d..0000000000 --- a/components/core/submodules/json +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03 diff --git a/components/core/submodules/log-surgeon b/components/core/submodules/log-surgeon deleted file mode 160000 index f801a3f369..0000000000 --- a/components/core/submodules/log-surgeon +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f801a3f369bb14919d82a928d9a3812006245aa0 diff --git a/components/core/submodules/outcome b/components/core/submodules/outcome deleted file mode 160000 index 571f9c930e..0000000000 --- a/components/core/submodules/outcome +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 571f9c930e672950e99d5d30f743603aaaf8014c diff --git a/components/core/submodules/simdjson b/components/core/submodules/simdjson deleted file mode 160000 index 6060be2fdf..0000000000 --- a/components/core/submodules/simdjson +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6060be2fdf62edf4a8f51a8b0883d57d09397b30 diff --git a/components/core/submodules/utfcpp b/components/core/submodules/utfcpp deleted file mode 160000 index b26a5f718f..0000000000 --- a/components/core/submodules/utfcpp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b26a5f718f4f370af1852a0d5c6ae8fa031ba7d0 diff --git a/components/core/submodules/yaml-cpp b/components/core/submodules/yaml-cpp deleted file mode 160000 index 0579ae3d97..0000000000 --- a/components/core/submodules/yaml-cpp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0579ae3d976091d7d664aa9d2527e0d0cff25763 diff --git a/components/core/submodules/ystdlib-cpp b/components/core/submodules/ystdlib-cpp deleted file mode 160000 index 2ac17579d1..0000000000 --- a/components/core/submodules/ystdlib-cpp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2ac17579d12c48e2f45d4688a10bcabb5becb466 diff --git a/components/core/tests/test-BoundedReader.cpp b/components/core/tests/test-BoundedReader.cpp index 9d1a9d2c0b..fae9980106 100644 --- a/components/core/tests/test-BoundedReader.cpp +++ b/components/core/tests/test-BoundedReader.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include "../src/clp/BoundedReader.hpp" #include "../src/clp/ErrorCode.hpp" diff --git a/components/core/tests/test-BufferedFileReader.cpp b/components/core/tests/test-BufferedFileReader.cpp index f182a777c8..5e960cb724 100644 --- a/components/core/tests/test-BufferedFileReader.cpp +++ b/components/core/tests/test-BufferedFileReader.cpp @@ -1,7 +1,7 @@ #include #include -#include +#include #include "../src/clp/BufferedFileReader.hpp" #include "../src/clp/FileReader.hpp" diff --git a/components/core/tests/test-EncodedVariableInterpreter.cpp b/components/core/tests/test-EncodedVariableInterpreter.cpp index 856b367851..4dcc6d399e 100644 --- a/components/core/tests/test-EncodedVariableInterpreter.cpp +++ b/components/core/tests/test-EncodedVariableInterpreter.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include "../src/clp/EncodedVariableInterpreter.hpp" #include "../src/clp/ir/types.hpp" diff --git a/components/core/tests/test-FileDescriptorReader.cpp b/components/core/tests/test-FileDescriptorReader.cpp index afc7a43565..b5cee12ed1 100644 --- a/components/core/tests/test-FileDescriptorReader.cpp +++ b/components/core/tests/test-FileDescriptorReader.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include #include #include "../src/clp/FileDescriptorReader.hpp" diff --git a/components/core/tests/test-Grep.cpp b/components/core/tests/test-Grep.cpp index 6d0603787f..f0bdb21dc3 100644 --- a/components/core/tests/test-Grep.cpp +++ b/components/core/tests/test-Grep.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include diff --git a/components/core/tests/test-MemoryMappedFile.cpp b/components/core/tests/test-MemoryMappedFile.cpp index 9cb855e0bd..9bd8fb6f31 100644 --- a/components/core/tests/test-MemoryMappedFile.cpp +++ b/components/core/tests/test-MemoryMappedFile.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include #include "../src/clp/FileReader.hpp" #include "../src/clp/ReaderInterface.hpp" diff --git a/components/core/tests/test-NetworkReader.cpp b/components/core/tests/test-NetworkReader.cpp index b382acc107..24c649f739 100644 --- a/components/core/tests/test-NetworkReader.cpp +++ b/components/core/tests/test-NetworkReader.cpp @@ -9,10 +9,10 @@ #include #include -#include +#include #include #include -#include +#include #include #include "../src/clp/CurlDownloadHandler.hpp" diff --git a/components/core/tests/test-ParserWithUserSchema.cpp b/components/core/tests/test-ParserWithUserSchema.cpp index 3689c69e85..4216fd7702 100644 --- a/components/core/tests/test-ParserWithUserSchema.cpp +++ b/components/core/tests/test-ParserWithUserSchema.cpp @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include "../src/clp/clp/run.hpp" diff --git a/components/core/tests/test-SQLiteDB.cpp b/components/core/tests/test-SQLiteDB.cpp index d63098bad6..cf2b338ded 100644 --- a/components/core/tests/test-SQLiteDB.cpp +++ b/components/core/tests/test-SQLiteDB.cpp @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include diff --git a/components/core/tests/test-Segment.cpp b/components/core/tests/test-Segment.cpp index c10844a9e8..ffed996ade 100644 --- a/components/core/tests/test-Segment.cpp +++ b/components/core/tests/test-Segment.cpp @@ -1,7 +1,7 @@ #include #include -#include +#include #include "../src/clp/streaming_archive/reader/Segment.hpp" #include "../src/clp/streaming_archive/writer/Segment.hpp" diff --git a/components/core/tests/test-Stopwatch.cpp b/components/core/tests/test-Stopwatch.cpp index 5f7d6bd710..e5f6f5a0af 100644 --- a/components/core/tests/test-Stopwatch.cpp +++ b/components/core/tests/test-Stopwatch.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include "../src/clp/Stopwatch.hpp" diff --git a/components/core/tests/test-StreamingCompression.cpp b/components/core/tests/test-StreamingCompression.cpp index 9b7125f961..ee67b843cd 100644 --- a/components/core/tests/test-StreamingCompression.cpp +++ b/components/core/tests/test-StreamingCompression.cpp @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include diff --git a/components/core/tests/test-TimestampPattern.cpp b/components/core/tests/test-TimestampPattern.cpp index 0b9dc54cf4..85acfbe467 100644 --- a/components/core/tests/test-TimestampPattern.cpp +++ b/components/core/tests/test-TimestampPattern.cpp @@ -1,4 +1,4 @@ -#include +#include #include "../src/clp/TimestampPattern.hpp" diff --git a/components/core/tests/test-Utils.cpp b/components/core/tests/test-Utils.cpp index 21d070f926..ce64d6ca01 100644 --- a/components/core/tests/test-Utils.cpp +++ b/components/core/tests/test-Utils.cpp @@ -8,7 +8,7 @@ #include #include -#include +#include #include "../src/clp/Utils.hpp" diff --git a/components/core/tests/test-clp_s-end_to_end.cpp b/components/core/tests/test-clp_s-end_to_end.cpp index b2f9b1b747..bd533159d2 100644 --- a/components/core/tests/test-clp_s-end_to_end.cpp +++ b/components/core/tests/test-clp_s-end_to_end.cpp @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include "../src/clp_s/InputConfig.hpp" diff --git a/components/core/tests/test-clp_s-search.cpp b/components/core/tests/test-clp_s-search.cpp index 0c6cc26dd2..57c0e573d6 100644 --- a/components/core/tests/test-clp_s-search.cpp +++ b/components/core/tests/test-clp_s-search.cpp @@ -9,9 +9,9 @@ #include #include -#include +#include #include -#include +#include #include "../src/clp_s/ArchiveReader.hpp" #include "../src/clp_s/InputConfig.hpp" diff --git a/components/core/tests/test-encoding_methods.cpp b/components/core/tests/test-encoding_methods.cpp index 9ff5f58f2f..444682fe3d 100644 --- a/components/core/tests/test-encoding_methods.cpp +++ b/components/core/tests/test-encoding_methods.cpp @@ -1,4 +1,4 @@ -#include +#include #include "../src/clp/ffi/encoding_methods.hpp" #include "../src/clp/ir/types.hpp" diff --git a/components/core/tests/test-ffi_IrUnitHandlerInterface.cpp b/components/core/tests/test-ffi_IrUnitHandlerInterface.cpp index 3b5763153a..01c750d8d4 100644 --- a/components/core/tests/test-ffi_IrUnitHandlerInterface.cpp +++ b/components/core/tests/test-ffi_IrUnitHandlerInterface.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include "../src/clp/ffi/ir_stream/decoding_methods.hpp" #include "../src/clp/ffi/ir_stream/IrUnitHandlerInterface.hpp" diff --git a/components/core/tests/test-ffi_KeyValuePairLogEvent.cpp b/components/core/tests/test-ffi_KeyValuePairLogEvent.cpp index 5c892e17ea..6a2f07f111 100644 --- a/components/core/tests/test-ffi_KeyValuePairLogEvent.cpp +++ b/components/core/tests/test-ffi_KeyValuePairLogEvent.cpp @@ -10,8 +10,9 @@ #include #include -#include -#include +#include +#include +#include #include "../src/clp/ffi/encoding_methods.hpp" #include "../src/clp/ffi/KeyValuePairLogEvent.hpp" diff --git a/components/core/tests/test-ffi_SchemaTree.cpp b/components/core/tests/test-ffi_SchemaTree.cpp index 2808e03c04..4cf67e32a7 100644 --- a/components/core/tests/test-ffi_SchemaTree.cpp +++ b/components/core/tests/test-ffi_SchemaTree.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include "../src/clp/ffi/SchemaTree.hpp" diff --git a/components/core/tests/test-hash_utils.cpp b/components/core/tests/test-hash_utils.cpp index 9e5aacd479..279ad4dde3 100644 --- a/components/core/tests/test-hash_utils.cpp +++ b/components/core/tests/test-hash_utils.cpp @@ -1,7 +1,7 @@ #include #include -#include +#include #include "../src/clp/ErrorCode.hpp" #include "../src/clp/hash_utils.hpp" diff --git a/components/core/tests/test-ir_encoding_methods.cpp b/components/core/tests/test-ir_encoding_methods.cpp index c645b3a136..7c4d0fa194 100644 --- a/components/core/tests/test-ir_encoding_methods.cpp +++ b/components/core/tests/test-ir_encoding_methods.cpp @@ -12,9 +12,9 @@ #include #include -#include -#include +#include #include +#include #include "../src/clp/BufferReader.hpp" #include "../src/clp/ErrorCode.hpp" diff --git a/components/core/tests/test-ir_parsing.cpp b/components/core/tests/test-ir_parsing.cpp index 1289e7c90e..b0b153c4e8 100644 --- a/components/core/tests/test-ir_parsing.cpp +++ b/components/core/tests/test-ir_parsing.cpp @@ -1,4 +1,4 @@ -#include +#include #include "../src/clp/ir/parsing.hpp" #include "../src/clp/ir/types.hpp" diff --git a/components/core/tests/test-ir_serializer.cpp b/components/core/tests/test-ir_serializer.cpp index 488f91d365..77b55ed342 100644 --- a/components/core/tests/test-ir_serializer.cpp +++ b/components/core/tests/test-ir_serializer.cpp @@ -5,7 +5,7 @@ #include #include -#include +#include #include "../src/clp/ffi/ir_stream/decoding_methods.hpp" #include "../src/clp/ir/constants.hpp" diff --git a/components/core/tests/test-kql.cpp b/components/core/tests/test-kql.cpp index 4d55bb556c..0f52b6b6bd 100644 --- a/components/core/tests/test-kql.cpp +++ b/components/core/tests/test-kql.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include diff --git a/components/core/tests/test-main.cpp b/components/core/tests/test-main.cpp index 9d9645b0cf..4ed06df1f7 100644 --- a/components/core/tests/test-main.cpp +++ b/components/core/tests/test-main.cpp @@ -1,2 +1,2 @@ #define CATCH_CONFIG_MAIN -#include +#include diff --git a/components/core/tests/test-math_utils.cpp b/components/core/tests/test-math_utils.cpp index b8bf6d12da..370d31d592 100644 --- a/components/core/tests/test-math_utils.cpp +++ b/components/core/tests/test-math_utils.cpp @@ -1,4 +1,4 @@ -#include +#include #include "../src/clp/math_utils.hpp" diff --git a/components/core/tests/test-query_methods.cpp b/components/core/tests/test-query_methods.cpp index b9b54a35e0..fa2163adf8 100644 --- a/components/core/tests/test-query_methods.cpp +++ b/components/core/tests/test-query_methods.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include "../src/clp/ffi/encoding_methods.hpp" #include "../src/clp/ffi/search/ExactVariableToken.hpp" diff --git a/components/core/tests/test-regex_utils.cpp b/components/core/tests/test-regex_utils.cpp index 38d401b25d..acf05ba775 100644 --- a/components/core/tests/test-regex_utils.cpp +++ b/components/core/tests/test-regex_utils.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include #include diff --git a/components/core/tests/test-sql.cpp b/components/core/tests/test-sql.cpp index 90e58eeca0..b3459a0e07 100644 --- a/components/core/tests/test-sql.cpp +++ b/components/core/tests/test-sql.cpp @@ -1,7 +1,7 @@ #include #include -#include +#include #include "../src/clp_s/search/ast/EmptyExpr.hpp" #include "../src/clp_s/search/sql/sql.hpp" diff --git a/components/core/tests/test-string_utils.cpp b/components/core/tests/test-string_utils.cpp index e55278d351..e57c83053b 100644 --- a/components/core/tests/test-string_utils.cpp +++ b/components/core/tests/test-string_utils.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include using clp::string_utils::clean_up_wildcard_search_string; diff --git a/components/core/tests/test-utf8_utils.cpp b/components/core/tests/test-utf8_utils.cpp index 7b45630002..b8e0e201b1 100644 --- a/components/core/tests/test-utf8_utils.cpp +++ b/components/core/tests/test-utf8_utils.cpp @@ -7,8 +7,8 @@ #include #include -#include -#include +#include +#include #include "../src/clp/ffi/utils.hpp" #include "../src/clp/utf8_utils.hpp" diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index 572b3ebfc6..808823f936 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -20,4 +20,6 @@ dnf install -y \ mariadb-connector-c-devel \ openssl-devel \ python3-pip \ + unzip \ + xz \ xz-devel diff --git a/components/core/tools/scripts/lib_install/ubuntu-focal/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-focal/install-prebuilt-packages.sh index 3ea3b3ed5f..c432ae618b 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-focal/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-focal/install-prebuilt-packages.sh @@ -28,6 +28,7 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ pkg-config \ python3 \ python3-pip \ + unzip \ zlib1g-dev # Add alternatives for cpp-10, gcc-10, and g++-10 diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index 402ee7e3a1..a0a8fdc94c 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -25,4 +25,5 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ python3 \ python3-pip \ python3-venv \ - software-properties-common + software-properties-common \ + unzip diff --git a/taskfile.yaml b/taskfile.yaml index 5fdec767eb..605a698c59 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -1,7 +1,7 @@ version: "3" includes: - deps: "taskfiles/deps.yaml" + deps: "taskfiles/deps/main.yaml" docs: "taskfiles/docs.yaml" lint: "taskfiles/lint.yaml" utils: "tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" @@ -27,7 +27,7 @@ vars: G_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/webui" # Taskfile paths - G_UTILS_TASKFILE: "{{.ROOT_DIR}}/tools/yscope-dev-utils/exports/taskfiles/utils.yaml" + G_UTILS_TASKFILE: "{{.ROOT_DIR}}/tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" # Versions G_PACKAGE_VERSION: "0.2.0-dev" diff --git a/taskfiles/deps.yaml b/taskfiles/deps.yaml deleted file mode 100644 index 027ac8faa6..0000000000 --- a/taskfiles/deps.yaml +++ /dev/null @@ -1,486 +0,0 @@ -version: "3" - -includes: - yscope-dev-utils: "../tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" - -vars: - # Utility script path - G_DEP_DOWNLOAD_SCRIPT: "{{.ROOT_DIR}}/tools/scripts/deps-download/download-dep.py" - - # Target checksum files - G_DEPS_CORE_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#core.md5" - G_DEPS_LOG_VIEWER_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#log-viewer.md5" - - # Submodule checksum files - G_ABSEIL_CPP_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#abseil-cpp.md5" - G_ANTLR4_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#antlr4.md5" - G_CATCH2_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#Catch2.md5" - G_DATE_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#date.md5" - G_JSON_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#json.md5" - G_LOG_SURGEON_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#log-surgeon.md5" - G_OUTCOME_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#outcome.md5" - G_SIMDJSON_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#simdjson.md5" - G_SQLITE3_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#sqlite3.md5" - G_UTFCPP_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#utfcpp.md5" - G_YAML_CPP_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#yaml-cpp.md5" - G_YSCOPE_LOG_VIEWER_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#yscope-log-viewer.md5" - G_YSTDLIB_CPP_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#ystdlib-cpp.md5" - -tasks: - default: - deps: - - "core" - - "log-viewer" - - core: - vars: - sources: - - "{{.G_ABSEIL_CPP_CHECKSUM_FILE}}" - - "{{.G_ANTLR4_CHECKSUM_FILE}}" - - "{{.G_CATCH2_CHECKSUM_FILE}}" - - "{{.G_DATE_CHECKSUM_FILE}}" - - "{{.G_JSON_CHECKSUM_FILE}}" - - "{{.G_LOG_SURGEON_CHECKSUM_FILE}}" - - "{{.G_OUTCOME_CHECKSUM_FILE}}" - - "{{.G_SIMDJSON_CHECKSUM_FILE}}" - - "{{.G_SQLITE3_CHECKSUM_FILE}}" - - "{{.G_UTFCPP_CHECKSUM_FILE}}" - - "{{.G_YAML_CPP_CHECKSUM_FILE}}" - - "{{.G_YSTDLIB_CPP_CHECKSUM_FILE}}" - generates: ["{{.G_DEPS_CORE_CHECKSUM_FILE}}"] - deps: ["all-internal-deps"] - cmds: - - >- - cat - "{{.G_ABSEIL_CPP_CHECKSUM_FILE}}" - "{{.G_ANTLR4_CHECKSUM_FILE}}" - "{{.G_CATCH2_CHECKSUM_FILE}}" - "{{.G_DATE_CHECKSUM_FILE}}" - "{{.G_JSON_CHECKSUM_FILE}}" - "{{.G_LOG_SURGEON_CHECKSUM_FILE}}" - "{{.G_OUTCOME_CHECKSUM_FILE}}" - "{{.G_SIMDJSON_CHECKSUM_FILE}}" - "{{.G_SQLITE3_CHECKSUM_FILE}}" - "{{.G_UTFCPP_CHECKSUM_FILE}}" - "{{.G_YAML_CPP_CHECKSUM_FILE}}" - "{{.G_YSTDLIB_CPP_CHECKSUM_FILE}}" - >> "{{.G_DEPS_CORE_CHECKSUM_FILE}}" - - log-viewer: - cmds: - - task: "yscope-dev-utils:remote:download-and-extract-tar" - vars: - CHECKSUM_FILE: "{{.G_DEPS_LOG_VIEWER_CHECKSUM_FILE}}" - FILE_SHA256: "de0600b505545f1bceb7ff5725941035f7c1dc875d08249d9b6d679e3ba77f26" - OUTPUT_DIR: "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/yscope-log-viewer" - TAR_FILE: "{{.G_BUILD_DIR}}/yscope-log-viewer.tar.gz" - URL: "https://github.com/y-scope/yscope-log-viewer/archive/969ff35.tar.gz" - - # NOTE: `git submodule update` doesn't support parallel invocations, so we can't use it to - # download submodules in parallel. This means: - # 1. the submodule tasks cannot be specified as a list of `deps`, because `task` always runs - # `deps` tasks in parallel. - # 2. we need to force higher level tasks, like `core` and `log-viewer`, to run their submodule - # download tasks serially. - # - # As a solution, the `all-internal-deps` task below: - # 1. uses `cmds` to sequentially download `ALL` submodules required by the clp-package. - # 2. uses `run: once`, so it executes once even if multiple higher level targets depend on it. - # - # All higher level tasks must depend on this task instead of depending on the individual - # submodule tasks in this file. - all-internal-deps: - internal: true - run: "once" - cmds: - - task: "abseil-cpp" - - task: "antlr4" - - task: "Catch2" - - task: "date" - - task: "json" - - task: "log-surgeon" - - task: "outcome" - - task: "simdjson" - - task: "sqlite3" - - task: "utfcpp" - - task: "yaml-cpp" - - task: "ystdlib-cpp" - - abseil-cpp: - internal: true - vars: - CHECKSUM_FILE: "{{.G_ABSEIL_CPP_CHECKSUM_FILE}}" - DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/abseil-cpp" - sources: - - "{{.G_DEP_DOWNLOAD_SCRIPT}}" - - "{{.G_UTILS_TASKFILE}}" - - "{{.ROOT_DIR}}/taskfile.yaml" - - "{{.TASKFILE}}" - generates: ["{{.CHECKSUM_FILE}}"] - deps: - - ":init" - - task: ":utils:checksum:validate" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - cmds: - - task: "download-dependency" - vars: - DEST: "{{.DEST}}" - FLAGS: "--extract" - SRC_NAME: "abseil-cpp-20230802.1" - SRC_URL: "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.zip" - # This command must be last - - task: ":utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - - antlr4: - internal: true - vars: - CHECKSUM_FILE: "{{.G_ANTLR4_CHECKSUM_FILE}}" - DEST_DIR: "{{.G_CORE_COMPONENT_DIR}}/third-party/antlr" - DEST: "{{.DEST_DIR}}/antlr-4.13.1-complete.jar" - sources: - - "{{.G_DEP_DOWNLOAD_SCRIPT}}" - - "{{.G_UTILS_TASKFILE}}" - - "{{.ROOT_DIR}}/taskfile.yaml" - - "{{.TASKFILE}}" - generates: ["{{.CHECKSUM_FILE}}"] - deps: - - ":init" - - task: ":utils:checksum:validate" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST_DIR}}"] - cmds: - - task: "download-dependency" - vars: - DEST: "{{.DEST}}" - FLAGS: "--no-submodule" - SRC_NAME: "antlr-4.13.1-complete.jar" - SRC_URL: "https://www.antlr.org/download/antlr-4.13.1-complete.jar" - # This command must be last - - task: ":utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST_DIR}}"] - - Catch2: - internal: true - vars: - CHECKSUM_FILE: "{{.G_CATCH2_CHECKSUM_FILE}}" - DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/Catch2" - sources: - - "{{.G_DEP_DOWNLOAD_SCRIPT}}" - - "{{.G_UTILS_TASKFILE}}" - - "{{.ROOT_DIR}}/taskfile.yaml" - - "{{.TASKFILE}}" - generates: ["{{.CHECKSUM_FILE}}"] - deps: - - ":init" - - task: ":utils:checksum:validate" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - cmds: - - task: "download-dependency" - vars: - DEST: "{{.DEST}}" - FLAGS: "--extract" - SRC_NAME: "Catch2-2.13.7" - SRC_URL: "https://github.com/catchorg/Catch2/archive/refs/tags/v2.13.7.zip" - # This command must be last - - task: ":utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - - date: - internal: true - vars: - CHECKSUM_FILE: "{{.G_DATE_CHECKSUM_FILE}}" - DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/date" - sources: - - "{{.G_DEP_DOWNLOAD_SCRIPT}}" - - "{{.G_UTILS_TASKFILE}}" - - "{{.ROOT_DIR}}/taskfile.yaml" - - "{{.TASKFILE}}" - generates: ["{{.CHECKSUM_FILE}}"] - deps: - - ":init" - - task: ":utils:checksum:validate" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - cmds: - - task: "download-dependency" - vars: - DEST: "{{.DEST}}" - FLAGS: "--extract" - SRC_NAME: "date-3.0.1" - SRC_URL: "https://github.com/HowardHinnant/date/archive/refs/tags/v3.0.1.zip" - # This command must be last - - task: ":utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - - json: - internal: true - vars: - CHECKSUM_FILE: "{{.G_JSON_CHECKSUM_FILE}}" - DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/json" - sources: - - "{{.G_DEP_DOWNLOAD_SCRIPT}}" - - "{{.G_UTILS_TASKFILE}}" - - "{{.ROOT_DIR}}/taskfile.yaml" - - "{{.TASKFILE}}" - generates: ["{{.CHECKSUM_FILE}}"] - deps: - - ":init" - - task: ":utils:checksum:validate" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - cmds: - - task: "download-dependency" - vars: - DEST: "{{.DEST}}" - FLAGS: "--extract" - SRC_NAME: "json-3.11.3" - SRC_URL: "https://github.com/nlohmann/json/archive/refs/tags/v3.11.3.zip" - # This command must be last - - task: ":utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - - log-surgeon: - internal: true - vars: - CHECKSUM_FILE: "{{.G_LOG_SURGEON_CHECKSUM_FILE}}" - DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/log-surgeon" - sources: - - "{{.G_DEP_DOWNLOAD_SCRIPT}}" - - "{{.G_UTILS_TASKFILE}}" - - "{{.ROOT_DIR}}/taskfile.yaml" - - "{{.TASKFILE}}" - generates: ["{{.CHECKSUM_FILE}}"] - deps: - - ":init" - - task: ":utils:checksum:validate" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - cmds: - - task: "download-dependency" - vars: - DEST: "{{.DEST}}" - FLAGS: "--extract" - SRC_NAME: "log-surgeon-f801a3f369bb14919d82a928d9a3812006245aa0" - SRC_URL: "https://github.com/y-scope/log-surgeon/archive/f801a3f.zip" - # This command must be last - - task: ":utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - - outcome: - internal: true - vars: - CHECKSUM_FILE: "{{.G_OUTCOME_CHECKSUM_FILE}}" - DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/outcome" - sources: - - "{{.G_DEP_DOWNLOAD_SCRIPT}}" - - "{{.G_UTILS_TASKFILE}}" - - "{{.ROOT_DIR}}/taskfile.yaml" - - "{{.TASKFILE}}" - generates: ["{{.CHECKSUM_FILE}}"] - deps: - - ":init" - - task: ":utils:checksum:validate" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - cmds: - - task: "download-dependency" - vars: - DEST: "{{.DEST}}" - FLAGS: "--extract" - SRC_NAME: "outcome-2.2.9" - SRC_URL: "https://github.com/ned14/outcome/archive/refs/tags/v2.2.9.zip" - # This command must be last - - task: ":utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - - simdjson: - internal: true - vars: - CHECKSUM_FILE: "{{.G_SIMDJSON_CHECKSUM_FILE}}" - DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/simdjson" - sources: - - "{{.G_DEP_DOWNLOAD_SCRIPT}}" - - "{{.G_UTILS_TASKFILE}}" - - "{{.ROOT_DIR}}/taskfile.yaml" - - "{{.TASKFILE}}" - generates: ["{{.CHECKSUM_FILE}}"] - deps: - - ":init" - - task: ":utils:checksum:validate" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - cmds: - - task: "download-dependency" - vars: - DEST: "{{.DEST}}" - FLAGS: "--extract" - SRC_NAME: "simdjson-3.6.3" - SRC_URL: "https://github.com/simdjson/simdjson/archive/refs/tags/v3.6.3.zip" - # This command must be last - - task: ":utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - - # We don't use a git submodule for sqlite3 since that would require building the sqlite - # amalgamation - sqlite3: - internal: true - vars: - CHECKSUM_FILE: "{{.G_SQLITE3_CHECKSUM_FILE}}" - DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/sqlite3" - sources: - - "{{.G_DEP_DOWNLOAD_SCRIPT}}" - - "{{.G_UTILS_TASKFILE}}" - - "{{.ROOT_DIR}}/taskfile.yaml" - - "{{.TASKFILE}}" - generates: ["{{.CHECKSUM_FILE}}"] - deps: - - ":init" - - task: ":utils:checksum:validate" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - cmds: - - task: "download-dependency" - vars: - DEST: "{{.DEST}}" - FLAGS: "--extract --no-submodule" - SRC_NAME: "sqlite-amalgamation-3360000" - SRC_URL: "https://www.sqlite.org/2021/sqlite-amalgamation-3360000.zip" - # This command must be last - - task: ":utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - - utfcpp: - internal: true - vars: - CHECKSUM_FILE: "{{.G_UTFCPP_CHECKSUM_FILE}}" - DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/utfcpp" - sources: - - "{{.G_DEP_DOWNLOAD_SCRIPT}}" - - "{{.G_UTILS_TASKFILE}}" - - "{{.ROOT_DIR}}/taskfile.yaml" - - "{{.TASKFILE}}" - generates: ["{{.CHECKSUM_FILE}}"] - deps: - - ":init" - - task: ":utils:checksum:validate" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - cmds: - - task: "download-dependency" - vars: - DEST: "{{.DEST}}" - FLAGS: "--extract" - SRC_NAME: "utfcpp-4.0.6" - SRC_URL: "https://github.com/nemtrif/utfcpp/archive/refs/tags/v4.0.6.zip" - # This command must be last - - task: ":utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - - yaml-cpp: - internal: true - vars: - CHECKSUM_FILE: "{{.G_YAML_CPP_CHECKSUM_FILE}}" - DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/yaml-cpp" - sources: - - "{{.G_DEP_DOWNLOAD_SCRIPT}}" - - "{{.G_UTILS_TASKFILE}}" - - "{{.ROOT_DIR}}/taskfile.yaml" - - "{{.TASKFILE}}" - generates: ["{{.CHECKSUM_FILE}}"] - deps: - - ":init" - - task: ":utils:checksum:validate" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - cmds: - - task: "download-dependency" - vars: - DEST: "{{.DEST}}" - FLAGS: "--extract" - SRC_NAME: "yaml-cpp-yaml-cpp-0.7.0" - SRC_URL: "https://github.com/jbeder/yaml-cpp/archive/refs/tags/yaml-cpp-0.7.0.zip" - # This command must be last - - task: ":utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - - ystdlib-cpp: - internal: true - vars: - CHECKSUM_FILE: "{{.G_YSTDLIB_CPP_CHECKSUM_FILE}}" - DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/ystdlib-cpp" - sources: - - "{{.G_DEP_DOWNLOAD_SCRIPT}}" - - "{{.G_UTILS_TASKFILE}}" - - "{{.ROOT_DIR}}/taskfile.yaml" - - "{{.TASKFILE}}" - generates: ["{{.CHECKSUM_FILE}}"] - deps: - - ":init" - - task: ":utils:checksum:validate" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - cmds: - - task: "download-dependency" - vars: - DEST: "{{.DEST}}" - FLAGS: "--extract" - SRC_NAME: "ystdlib-cpp-2ac17579d12c48e2f45d4688a10bcabb5becb466" - SRC_URL: "https://github.com/y-scope/ystdlib-cpp/archive/2ac1757.zip" - # This command must be last - - task: ":utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.DEST}}"] - - - download-dependency: - internal: true - label: "{{.TASK}}-{{.SRC_NAME}}" - requires: - vars: ["DEST", "FLAGS", "SRC_NAME", "SRC_URL"] - deps: - - ":init" - cmds: - - >- - python3 "{{.G_DEP_DOWNLOAD_SCRIPT}}" - "{{.SRC_URL}}" - {{.SRC_NAME}} - "{{.DEST}}" - {{.FLAGS}} diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml new file mode 100644 index 0000000000..75c4f92b21 --- /dev/null +++ b/taskfiles/deps/main.yaml @@ -0,0 +1,303 @@ +version: "3" + +includes: + utils: "./utils.yaml" + yscope-dev-utils: "../../tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" + +vars: + # Directories + G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" + G_DEPS_CORE_DIR: "{{.G_DEPS_DIR}}/core" + G_DEPS_CORE_CHECKSUMS_DIR: "{{.G_DEPS_CORE_DIR}}/checksums" + + # Checksum file paths + G_DEPS_CORE_CHECKSUM_FILE: "{{.G_DEPS_CORE_CHECKSUMS_DIR}}/all.md5" + G_DEPS_LOG_VIEWER_CHECKSUM_FILE: "{{.G_DEPS_DIR}}/log-viewer.md5" + + # NOTE: This must be kept in-sync with its usage in components/core/CMakeLists.txt + G_DEPS_CORE_CMAKE_SETTINGS_DIR: "{{.G_DEPS_CORE_DIR}}/cmake-settings" + +tasks: + default: + deps: + - "core" + - "log-viewer" + + core: + deps: + - "utils:clean-outdated-core-checksum-files" + - "utils:init" + cmds: + - "rm -rf '{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}'" + - "mkdir -p '{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}'" + - task: "yscope-dev-utils:cmake:install-deps-and-generate-settings" + vars: + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}" + DEP_TASK: "core-all-parallel" + - task: "utils:combine-core-checksum-files" + + log-viewer: + deps: + - "utils:init" + cmds: + - task: "yscope-dev-utils:remote:download-and-extract-tar" + vars: + CHECKSUM_FILE: "{{.G_DEPS_LOG_VIEWER_CHECKSUM_FILE}}" + FILE_SHA256: "de0600b505545f1bceb7ff5725941035f7c1dc875d08249d9b6d679e3ba77f26" + OUTPUT_DIR: "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/yscope-log-viewer" + TAR_FILE: "{{.G_BUILD_DIR}}/yscope-log-viewer.tar.gz" + URL: "https://github.com/y-scope/yscope-log-viewer/archive/969ff35.tar.gz" + + core-all-parallel: + internal: true + run: "once" + deps: + - task: "absl" + - task: "antlr" + - task: "catch2" + - task: "date" + - task: "log-surgeon" + - task: "nlohmann_json" + - task: "outcome" + - task: "simdjson" + - task: "sqlite3" + - task: "utfcpp" + - task: "yaml-cpp" + - task: "ystdlib" + + absl: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_CXX_STANDARD=20" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + LIB_NAME: "absl" + TARBALL_SHA256: "987ce98f02eefbaf930d6e38ab16aa05737234d7afbab2d5c4ea7adbe50c28ed" + TARBALL_URL: "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz" + + antlr: + internal: true + vars: + DEP_NAME: "antlr" + + # Paths + CHECKSUM_FILE: "{{.G_DEPS_CORE_CHECKSUMS_DIR}}/{{.DEP_NAME}}.md5" + OUTPUT_FILE: "{{.G_DEPS_CORE_DIR}}/antlr-4.13.1-complete.jar" + run: "once" + deps: + - task: "yscope-dev-utils:checksum:validate" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: + - "{{.OUTPUT_FILE}}" + - task: "utils:init" + cmds: + - task: "yscope-dev-utils:remote:curl" + vars: + FILE_SHA256: "bc13a9c57a8dd7d5196888211e5ede657cb64a3ce968608697e4f668251a8487" + OUTPUT_FILE: "{{.OUTPUT_FILE}}" + URL: "https://www.antlr.org/download/antlr-4.13.1-complete.jar" + - >- + echo "set( + ANTLR_EXECUTABLE \"{{.OUTPUT_FILE}}\" + )" > "{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}/{{.DEP_NAME}}.cmake" + + # This command must be last + - task: "yscope-dev-utils:checksum:compute" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: + - "{{.OUTPUT_FILE}}" + + catch2: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DBUILD_TESTING=OFF" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + LIB_NAME: "Catch2" + TARBALL_SHA256: "3cdb4138a072e4c0290034fe22d9f0a80d3bcfb8d7a8a5c49ad75d3a5da24fae" + TARBALL_URL: "https://github.com/catchorg/Catch2/archive/refs/tags/v2.13.7.tar.gz" + + date: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + LIB_NAME: "date" + TARBALL_SHA256: "7a390f200f0ccd207e8cff6757e04817c1a0aec3e327b006b7eb451c57ee3538" + TARBALL_URL: "https://github.com/HowardHinnant/date/archive/refs/tags/v3.0.1.tar.gz" + + log-surgeon: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + LIB_NAME: "log_surgeon" + TARBALL_SHA256: "169b56714b620c3f8f07bf7975aba8573b9bd25e423e5c4f0cc9b52450a2403d" + TARBALL_URL: "https://github.com/y-scope/log-surgeon/archive/f801a3f.tar.gz" + + nlohmann_json: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DJSON_BuildTests=OFF" + LIB_NAME: "nlohmann_json" + TARBALL_SHA256: "0d8ef5af7f9794e3263480193c491549b2ba6cc74bb018906202ada498a79406" + + # NOTE: We use the GitHub-generated source tarball for this version rather than the + # release tarball, since the latter is served from githubusercontent.com which is blocked + # by some developers' firewalls. The contents of the former are a superset of the latter. + TARBALL_URL: "https://github.com/nlohmann/json/archive/refs/tags/v3.11.3.tar.gz" + + outcome: + internal: true + vars: + LIB_NAME: "outcome" + + # Paths + INSTALL_INCLUDE_DIR: "{{.G_DEPS_CORE_DIR}}/{{.LIB_NAME}}-install/include" + INSTALL_SYMLINK: "{{.INSTALL_INCLUDE_DIR}}/{{.LIB_NAME}}" + SRC_DIR: "{{.G_DEPS_CORE_DIR}}/{{.LIB_NAME}}-src" + run: "once" + deps: + - task: "utils:init" + cmds: + - task: "yscope-dev-utils:remote:download-and-extract-tar" + vars: + CHECKSUM_FILE: "{{.G_DEPS_CORE_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" + FILE_SHA256: "2840e403b1d7a0d3a75ecc4df574ab0674284bb21d76ac5f817695f2b56905f2" + INCLUDE_PATTERNS: + - "*/single-header" + OUTPUT_DIR: "{{.SRC_DIR}}" + + # NOTE: We use the GitHub-generated source tarball for this version rather than the + # release tarball, since the latter is served from githubusercontent.com which is blocked + # by some developers' firewalls. The source files we use are identical between the two + # tarballs. + URL: "https://github.com/ned14/outcome/archive/refs/tags/v2.2.9.tar.gz" + - "mkdir -p '{{.INSTALL_INCLUDE_DIR}}'" + - "rm -f '{{.INSTALL_SYMLINK}}'" + - "ln -s '{{.SRC_DIR}}/single-header' '{{.INSTALL_SYMLINK}}'" + - >- + echo "set( + CLP_OUTCOME_INCLUDE_DIRECTORY \"{{.INSTALL_INCLUDE_DIR}}\" + )" > "{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}/{{.LIB_NAME}}.cmake" + + simdjson: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + LIB_NAME: "simdjson" + TARBALL_SHA256: "afd8201cfb1abe927737d876441bb1f21730a9ee6078a1b8c6174e6463981fa3" + TARBALL_URL: "https://github.com/simdjson/simdjson/archive/refs/tags/v3.6.3.tar.gz" + + sqlite3: + internal: true + vars: + LIB_NAME: "sqlite3" + + # Paths + EXTRACTION_DIR: "{{.G_DEPS_CORE_DIR}}/{{.LIB_NAME}}-src" + INSTALL_INCLUDE_DIR: "{{.G_DEPS_CORE_DIR}}/{{.LIB_NAME}}-install/include" + INSTALL_SYMLINK: "{{.INSTALL_INCLUDE_DIR}}/{{.LIB_NAME}}" + ZIP_FILENAME_STEM: "sqlite-amalgamation-3360000" + SRC_DIR: "{{.EXTRACTION_DIR}}/{{.ZIP_FILENAME_STEM}}" + run: "once" + deps: + - task: "utils:init" + cmds: + - task: "yscope-dev-utils:remote:download-and-extract-zip" + vars: + CHECKSUM_FILE: "{{.G_DEPS_CORE_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" + FILE_SHA256: "999826fe4c871f18919fdb8ed7ec9dd8217180854dd1fe21eea96aed36186729" + OUTPUT_DIR: "{{.EXTRACTION_DIR}}" + URL: "https://www.sqlite.org/2021/{{.ZIP_FILENAME_STEM}}.zip" + - "mkdir -p '{{.INSTALL_INCLUDE_DIR}}'" + - "rm -f '{{.INSTALL_SYMLINK}}'" + - "ln -s '{{.SRC_DIR}}' '{{.INSTALL_SYMLINK}}'" + - |- + echo "set(CLP_SQLITE3_SOURCE_DIRECTORY \"{{.SRC_DIR}}\")" \ + > "{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}/{{.LIB_NAME}}.cmake" + echo "set(CLP_SQLITE3_INCLUDE_DIRECTORY \"{{.INSTALL_INCLUDE_DIR}}\")" \ + >> "{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}/{{.LIB_NAME}}.cmake" + + utfcpp: + internal: true + vars: + LIB_NAME: "utfcpp" + UTFCPP_OUTPUT_DIR: "{{.G_DEPS_CORE_DIR}}/{{.LIB_NAME}}-src" + run: "once" + deps: + - task: "utils:init" + cmds: + - task: "yscope-dev-utils:remote:download-and-extract-tar" + vars: + CHECKSUM_FILE: "{{.G_DEPS_CORE_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" + FILE_SHA256: "6920a6a5d6a04b9a89b2a89af7132f8acefd46e0c2a7b190350539e9213816c0" + OUTPUT_DIR: "{{.UTFCPP_OUTPUT_DIR}}" + URL: "https://github.com/nemtrif/utfcpp/archive/refs/tags/v4.0.6.tar.gz" + - >- + echo "set( + CLP_UTFCPP_INCLUDE_DIRECTORY \"{{.UTFCPP_OUTPUT_DIR}}\" + )" > "{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}/{{.LIB_NAME}}.cmake" + + yaml-cpp: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DBUILD_TESTING=OFF" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + LIB_NAME: "yaml-cpp" + TARBALL_SHA256: "43e6a9fcb146ad871515f0d0873947e5d497a1c9c60c58cb102a97b47208b7c3" + TARBALL_URL: "https://github.com/jbeder/yaml-cpp/archive/refs/tags/yaml-cpp-0.7.0.tar.gz" + + ystdlib: + internal: true + vars: + LIB_NAME: "ystdlib" + YSTDLIB_OUTPUT_DIR: "{{.G_DEPS_CORE_DIR}}/{{.LIB_NAME}}-src" + run: "once" + deps: + - task: "utils:init" + cmds: + - task: "yscope-dev-utils:remote:download-and-extract-tar" + vars: + CHECKSUM_FILE: "{{.G_DEPS_CORE_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" + FILE_SHA256: "d47c703c0d1c5e755c1bfb984b68c85967683a09dadbe23772d9eb69f822d040" + OUTPUT_DIR: "{{.YSTDLIB_OUTPUT_DIR}}" + URL: "https://github.com/y-scope/ystdlib-cpp/archive/2ac1757.tar.gz" + - >- + echo "set( + CLP_YSTDLIB_SOURCE_DIRECTORY \"{{.YSTDLIB_OUTPUT_DIR}}\" + )" > "{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}/{{.LIB_NAME}}.cmake" diff --git a/taskfiles/deps/utils.yaml b/taskfiles/deps/utils.yaml new file mode 100644 index 0000000000..89287acbba --- /dev/null +++ b/taskfiles/deps/utils.yaml @@ -0,0 +1,79 @@ +version: "3" + +includes: + yscope-dev-utils: "../../tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" + +tasks: + # Cleans up the core checksum files any time the taskfiles change. This is to ensure that if any + # of core's dependencies are removed from the taskfiles, then the checksum files for the removed + # dependencies will also be removed. Since we can't easily remove just the checksum files for the + # removed dependencies, we remove all of them. + clean-outdated-core-checksum-files: + internal: true + run: "once" + sources: + - "{{.ROOT_DIR}}/Taskfile.yaml" + - "{{.ROOT_DIR}}/taskfiles/**/*.yaml" + - "{{.ROOT_DIR}}/tools/yscope-dev-utils/exports/taskfiles/**/*.yaml" + cmd: "rm -f '{{.G_DEPS_CORE_CHECKSUMS_DIR}}'/*" + + # Combines all of core's checksum files into `G_DEPS_CORE_CHECKSUM_FILE`. + combine-core-checksum-files: + internal: true + cmds: + - "rm -f '{{.G_DEPS_CORE_CHECKSUM_FILE}}'" + - |- + for file in "{{.G_DEPS_CORE_CHECKSUMS_DIR}}"/*.md5; do + if [[ "$file" != "{{.G_DEPS_CORE_CHECKSUM_FILE}}" ]]; then + cat "$file" >> "{{.G_DEPS_CORE_CHECKSUM_FILE}}" + fi + done + + init: + internal: true + run: "once" + cmds: + - "mkdir -p '{{.G_DEPS_DIR}}'" + - "mkdir -p '{{.G_DEPS_CORE_DIR}}'" + - "mkdir -p '{{.G_DEPS_CORE_CHECKSUMS_DIR}}'" + - "mkdir -p '{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}'" + + # Installs a CMake-based library from a remote tarball, generates a corresponding settings file, + # and computes a checksum for the installation directory. + # + # @param {string} LIB_NAME + # @param {string} TARBALL_SHA256 + # @param {string} TARBALL_URL + # @param {string[]} [CMAKE_GEN_ARGS] Any additional arguments to pass to the CMake generate + # command. + install-remote-cmake-lib: + internal: true + vars: + CMAKE_GEN_ARGS: + ref: "default (list) .CMAKE_GEN_ARGS" + INSTALL_PREFIX: "{{.G_DEPS_CORE_DIR}}/{{.LIB_NAME}}-install" + INSTALL_DIR_CHECKSUM_FILE: "{{.G_DEPS_CORE_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" + deps: + - task: "init" + - task: "yscope-dev-utils:checksum:validate" + vars: + CHECKSUM_FILE: "{{.INSTALL_DIR_CHECKSUM_FILE}}" + INCLUDE_PATTERNS: + - "{{.INSTALL_PREFIX}}" + cmds: + - task: "yscope-dev-utils:cmake:install-remote-tar" + vars: + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}" + FILE_SHA256: "{{.TARBALL_SHA256}}" + GEN_ARGS: + ref: ".CMAKE_GEN_ARGS" + NAME: "{{.LIB_NAME}}" + URL: "{{.TARBALL_URL}}" + WORK_DIR: "{{.G_DEPS_CORE_DIR}}" + + # This command must be last + - task: "yscope-dev-utils:checksum:compute" + vars: + CHECKSUM_FILE: "{{.INSTALL_DIR_CHECKSUM_FILE}}" + INCLUDE_PATTERNS: + - "{{.INSTALL_PREFIX}}" diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index e51528eea0..d9a85ce8da 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -118,8 +118,7 @@ tasks: components/package-template/src/etc \ docs \ taskfile.yaml \ - taskfiles/deps.yaml \ - taskfiles/lint.yaml + taskfiles check-cpp-format: sources: &cpp_source_files diff --git a/tools/scripts/deps-download/init.sh b/tools/scripts/deps-download/init.sh index 9395541250..486c5c4c11 100755 --- a/tools/scripts/deps-download/init.sh +++ b/tools/scripts/deps-download/init.sh @@ -11,7 +11,7 @@ project_root_dir="$script_dir/../../../" download_dep_script="$script_dir/download-dep.py" python3 "${download_dep_script}" \ - https://github.com/y-scope/yscope-dev-utils/archive/30640b0.zip \ - yscope-dev-utils-30640b0af5a8d344ae57afe02e8ae88a79233809 \ + https://github.com/y-scope/yscope-dev-utils/archive/e300d1b.zip \ + yscope-dev-utils-e300d1bab4c2f33cbf9ddf9f0c08185faf035070 \ "${project_root_dir}/tools/yscope-dev-utils" \ --extract diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils index 30640b0af5..e300d1bab4 160000 --- a/tools/yscope-dev-utils +++ b/tools/yscope-dev-utils @@ -1 +1 @@ -Subproject commit 30640b0af5a8d344ae57afe02e8ae88a79233809 +Subproject commit e300d1bab4c2f33cbf9ddf9f0c08185faf035070