Skip to content
9 changes: 7 additions & 2 deletions .github/workflows/pr-compile-check.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,13 @@ jobs:
- name: Setup environment
run: |
sudo apt-get update
sudo apt-get install -y gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev libluajit-5.1-dev
sudo apt-get install -y curl gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev libluajit-5.1-dev \
libnghttp2-dev libjemalloc-dev
sudo ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true
mkdir -p /tmp/libbacktrace/build && \
curl -L https://github.com/ianlancetaylor/libbacktrace/archive/8602fda.tar.gz | \
tar --strip-components=1 -xzC /tmp/libbacktrace/ && \
pushd /tmp/libbacktrace/build && ../configure && make && sudo make install && popd

- name: Checkout Fluent Bit code
uses: actions/checkout@v4
Expand All @@ -67,7 +72,7 @@ jobs:
CC: ${{ matrix.compiler }}
CXX: ${{ matrix.compiler }}
FLB_OPT: ${{ matrix.flb_option }}
GLOBAL_OPTS: "-DFLB_BACKTRACE=Off -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off"
GLOBAL_OPTS: "-DFLB_JEMALLOC=On -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off"

- name: Display dependencies w/ ldd
run: |
Expand Down
166 changes: 98 additions & 68 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,12 @@ option(FLB_RUN_LDCONFIG "Enable execution of ldconfig after installation" No)

# Prefer system libraries if available
option(FLB_PREFER_SYSTEM_LIBS "Prefer system libraries" No)
option(FLB_PREFER_SYSTEM_LIB_BACKTRACE "Prefer the libbacktrace system library" ${FLB_PREFER_SYSTEM_LIBS})
option(FLB_PREFER_SYSTEM_LIB_CARES "Prefer the libcares system library" ${FLB_PREFER_SYSTEM_LIBS})
option(FLB_PREFER_SYSTEM_LIB_JEMALLOC "Prefer the libjemalloc system library" ${FLB_PREFER_SYSTEM_LIBS})
option(FLB_PREFER_SYSTEM_LIB_KAFKA "Prefer the libkafka system library" ${FLB_PREFER_SYSTEM_LIBS})
option(FLB_PREFER_SYSTEM_LIB_LUAJIT "Prefer the libluajit system library" ${FLB_PREFER_SYSTEM_LIBS})
option(FLB_PREFER_SYSTEM_LIB_NGHTTP2 "Prefer the libnghttp2 system library" ${FLB_PREFER_SYSTEM_LIBS})

# Enable all features
if(FLB_ALL)
Expand Down Expand Up @@ -484,25 +489,28 @@ add_subdirectory(${FLB_PATH_LIB_CMETRICS} EXCLUDE_FROM_ALL)
add_subdirectory(${FLB_PATH_LIB_CTRACES} EXCLUDE_FROM_ALL)

# Nghttp2 options
FLB_OPTION(ENABLE_LIB_ONLY ON)
FLB_OPTION(ENABLE_STATIC_LIB ON)
FLB_OPTION(ENABLE_SHARED_LIB OFF)

FLB_DEFINITION(NGHTTP2_STATICLIB)

add_subdirectory(${FLB_PATH_LIB_NGHTTP2} EXCLUDE_FROM_ALL)
if(FLB_PREFER_SYSTEM_LIB_NGHTTP2)
find_package(PkgConfig)
pkg_check_modules(NGHTTP2 libnghttp2>=1.0.0)
endif()
if(NGHTTP2_FOUND)
include_directories(${NGHTTP2_INCLUDE_DIRS})
link_directories(${NGHTTP2_LIBRARY_DIRS})
else()
include(cmake/nghttp2.cmake)
endif()

# C-Ares (DNS library)
FLB_OPTION(CARES_STATIC ON)
FLB_OPTION(CARES_SHARED OFF)
FLB_OPTION(CARES_INSTALL OFF)
FLB_OPTION(CARES_BUILD_TESTS OFF)
FLB_OPTION(CARES_BUILD_TOOLS OFF)
if (FLB_SYSTEM_MACOS)
# macOS SDK always has <arpa/nameser.h>.
FLB_DEFINITION(CARES_HAVE_ARPA_NAMESER_H)
if(FLB_PREFER_SYSTEM_LIB_CARES)
find_package(PkgConfig)
pkg_check_modules(LIBCARES libcares>=1.18.0)
endif()
if(LIBCARES_FOUND)
include_directories(${LIBCARES_INCLUDE_DIRS})
link_directories(${LIBCARES_LIBRARY_DIRS})
else()
include(cmake/cares.cmake)
endif()
add_subdirectory(${FLB_PATH_LIB_CARES})# EXCLUDE_FROM_ALL)

# Chunk I/O
FLB_OPTION(CIO_LIB_STATIC ON)
Expand Down Expand Up @@ -598,6 +606,7 @@ if(FLB_WASM)
_Atomic int a;
return 0;
}" FLB_HAVE_STDATOMIC_H)
include(CheckIncludeFiles)
check_include_files(stdatomic.h FLB_HAVE_STDATOMIC_H)
if (FLB_HAVE_STDATOMIC_H)
enable_language (ASM)
Expand Down Expand Up @@ -902,72 +911,93 @@ endif()
# Memory Allocator
# ================
if(FLB_JEMALLOC AND ${CMAKE_SYSTEM_NAME} MATCHES "Linux")
FLB_DEFINITION(FLB_HAVE_JEMALLOC)
FLB_DEFINITION(JEMALLOC_MANGLE)
if(FLB_PREFER_SYSTEM_LIB_JEMALLOC)
find_package(PkgConfig)
pkg_check_modules(JEMALLOC jemalloc>=5.0.0)
endif()
if(JEMALLOC_FOUND)
include_directories(${JEMALLOC_INCLUDE_DIRS})
link_directories(${JEMALLOC_LIBRARY_DIRS})
else()
FLB_DEFINITION(JEMALLOC_MANGLE)

# Add support for options like page size, if empty we default it
if(NOT DEFINED FLB_JEMALLOC_OPTIONS OR "${FLB_JEMALLOC_OPTIONS}" STREQUAL "")
set(FLB_JEMALLOC_OPTIONS "--with-lg-quantum=3")
# Add support for options like page size, if empty we default it
if(NOT DEFINED FLB_JEMALLOC_OPTIONS OR "${FLB_JEMALLOC_OPTIONS}" STREQUAL "")
set(FLB_JEMALLOC_OPTIONS "--with-lg-quantum=3")
endif()
# Split into a list so CMake handles it correctly when passing to configure command
separate_arguments(FLB_JEMALLOC_OPTIONS_LIST UNIX_COMMAND ${FLB_JEMALLOC_OPTIONS})
message(STATUS "jemalloc configure: ${FLB_JEMALLOC_OPTIONS_LIST}")

# Link to Jemalloc as an external dependency
set(FLB_LIBJEMALLOC_PATH "${CMAKE_CURRENT_BINARY_DIR}/lib/libjemalloc_pic.a")
ExternalProject_Add(jemalloc
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/jemalloc-5.3.0
CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/lib/jemalloc-5.3.0/configure ${AUTOCONF_HOST_OPT} "${FLB_JEMALLOC_OPTIONS_LIST}" --prefix=<INSTALL_DIR>
CFLAGS=-std=gnu99\ -Wall\ -pipe\ -g3\ -O3\ -funroll-loops
BUILD_COMMAND ${EXTERNAL_BUILD_TOOL}
BUILD_BYPRODUCTS ${FLB_LIBJEMALLOC_PATH}
INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/
INSTALL_COMMAND ${EXTERNAL_BUILD_TOOL} install_lib_static install_include)
add_library(libjemalloc STATIC IMPORTED GLOBAL)
set_target_properties(libjemalloc PROPERTIES IMPORTED_LOCATION ${FLB_LIBJEMALLOC_PATH})
add_dependencies(libjemalloc jemalloc)
include_directories("${CMAKE_BINARY_DIR}/include/")
set(JEMALLOC_LIBRARIES "libjemalloc")
endif()
# Split into a list so CMake handles it correctly when passing to configure command
separate_arguments(FLB_JEMALLOC_OPTIONS_LIST UNIX_COMMAND ${FLB_JEMALLOC_OPTIONS})
message(STATUS "jemalloc configure: ${FLB_JEMALLOC_OPTIONS_LIST}")

# Link to Jemalloc as an external dependency
set(FLB_LIBJEMALLOC_PATH "${CMAKE_CURRENT_BINARY_DIR}/lib/libjemalloc_pic.a")
ExternalProject_Add(jemalloc
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/jemalloc-5.3.0
CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/lib/jemalloc-5.3.0/configure ${AUTOCONF_HOST_OPT} "${FLB_JEMALLOC_OPTIONS_LIST}" --prefix=<INSTALL_DIR>
CFLAGS=-std=gnu99\ -Wall\ -pipe\ -g3\ -O3\ -funroll-loops
BUILD_COMMAND ${EXTERNAL_BUILD_TOOL}
BUILD_BYPRODUCTS ${FLB_LIBJEMALLOC_PATH}
INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/
INSTALL_COMMAND ${EXTERNAL_BUILD_TOOL} install_lib_static install_include)
add_library(libjemalloc STATIC IMPORTED GLOBAL)
set_target_properties(libjemalloc PROPERTIES IMPORTED_LOCATION ${FLB_LIBJEMALLOC_PATH})
add_dependencies(libjemalloc jemalloc)
include_directories("${CMAKE_BINARY_DIR}/include/")
FLB_DEFINITION(FLB_HAVE_JEMALLOC)
else()
FLB_OPTION(FLB_JEMALLOC OFF)
endif()

# LibBacktrace (friendly stacktrace support)
# =========================================
if(FLB_BACKTRACE)
FLB_DEFINITION(FLB_HAVE_LIBBACKTRACE)
if (CMAKE_OSX_SYSROOT)
# From macOS Mojave, /usr/include does not store C SDK headers.
# For libbacktrace building on macOS, we have to tell C headers where they are located.
set(DEPS_C_COMPILER "${CMAKE_C_COMPILER} -isysroot ${CMAKE_OSX_SYSROOT}")
if(FLB_PREFER_SYSTEM_LIB_BACKTRACE)
find_path(LIBBACKTRACE_INCLUDE_DIRS NAMES backtrace.h)
find_library(LIBBACKTRACE_LIBRARIES NAMES backtrace)
endif()
if(LIBBACKTRACE_INCLUDE_DIRS AND LIBBACKTRACE_LIBRARIES)
message(STATUS "libbacktrace found (${LIBBACKTRACE_LIBRARIES})")
include_directories(${LIBBACKTRACE_INCLUDE_DIRS})
link_directories(${LIBBACKTRACE_LIBRARY_DIRS})
else()
set(DEPS_C_COMPILER "${CMAKE_C_COMPILER}")
message(STATUS "libbacktrace not found, building ourselves")
if (CMAKE_OSX_SYSROOT)
# From macOS Mojave, /usr/include does not store C SDK headers.
# For libbacktrace building on macOS, we have to tell C headers where they are located.
set(DEPS_C_COMPILER "${CMAKE_C_COMPILER} -isysroot ${CMAKE_OSX_SYSROOT}")
else()
set(DEPS_C_COMPILER "${CMAKE_C_COMPILER}")
endif()
set(FLB_LIBBACKTRACE_PATH "${CMAKE_CURRENT_BINARY_DIR}/backtrace-prefix/lib/libbacktrace.a")
ExternalProject_Add(backtrace
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/libbacktrace-8602fda/
CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/lib/libbacktrace-8602fda/configure ${AUTOCONF_HOST_OPT} --prefix=<INSTALL_DIR> --enable-shared=no --enable-static=yes
BUILD_COMMAND ${EXTERNAL_BUILD_TOOL}
BUILD_BYPRODUCTS ${FLB_LIBBACKTRACE_PATH}
INSTALL_COMMAND ${EXTERNAL_BUILD_TOOL} DESTDIR= install
)
add_library(libbacktrace STATIC IMPORTED GLOBAL)
set_target_properties(libbacktrace PROPERTIES IMPORTED_LOCATION ${FLB_LIBBACKTRACE_PATH})
add_dependencies(libbacktrace backtrace)
include_directories("${CMAKE_CURRENT_BINARY_DIR}/backtrace-prefix/include/")
set(LIBBACKTRACE_LIBRARIES "libbacktrace")
endif()
set(FLB_LIBBACKTRACE_PATH "${CMAKE_CURRENT_BINARY_DIR}/backtrace-prefix/lib/libbacktrace.a")
ExternalProject_Add(backtrace
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/libbacktrace-8602fda/
CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/lib/libbacktrace-8602fda/configure ${AUTOCONF_HOST_OPT} --prefix=<INSTALL_DIR> --enable-shared=no --enable-static=yes
BUILD_COMMAND ${EXTERNAL_BUILD_TOOL}
BUILD_BYPRODUCTS ${FLB_LIBBACKTRACE_PATH}
INSTALL_COMMAND ${EXTERNAL_BUILD_TOOL} DESTDIR= install
)
add_library(libbacktrace STATIC IMPORTED GLOBAL)
set_target_properties(libbacktrace PROPERTIES IMPORTED_LOCATION ${FLB_LIBBACKTRACE_PATH})
add_dependencies(libbacktrace backtrace)
include_directories("${CMAKE_CURRENT_BINARY_DIR}/backtrace-prefix/include/")
FLB_DEFINITION(FLB_HAVE_LIBBACKTRACE)
endif()

if(FLB_IN_KAFKA OR FLB_OUT_KAFKA)
FLB_OPTION(RDKAFKA_BUILD_STATIC On)
FLB_OPTION(RDKAFKA_BUILD_EXAMPLES Off)
FLB_OPTION(RDKAFKA_BUILD_TESTS Off)
FLB_OPTION(ENABLE_LZ4_EXT Off)

# disable Curl
if (FLB_SYSTEM_MACOS)
FLB_OPTION(WITH_CURL Off)
endif()

add_subdirectory(${FLB_PATH_LIB_RDKAFKA} EXCLUDE_FROM_ALL)
if(FLB_PREFER_SYSTEM_LIB_KAFKA)
find_package(PkgConfig)
pkg_check_modules(KAFKA rdkafka>=2.3.0)
endif()
if(KAFKA_FOUND)
include_directories(${KAFKA_INCLUDE_DIRS})
link_directories(${KAFKA_LIBRARY_DIRS})
else()
include(cmake/kafka.cmake)
endif()
endif()

# Onigmo (Regex Engine) options
Expand Down
20 changes: 20 additions & 0 deletions cmake/cares.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# cares cmake
FLB_OPTION(CARES_STATIC ON)
FLB_OPTION(CARES_SHARED OFF)
FLB_OPTION(CARES_INSTALL OFF)
FLB_OPTION(CARES_BUILD_TESTS OFF)
FLB_OPTION(CARES_BUILD_TOOLS OFF)

if (FLB_SYSTEM_MACOS)
# macOS SDK always has <arpa/nameser.h>.
FLB_DEFINITION(CARES_HAVE_ARPA_NAMESER_H)
endif()

include_directories(
${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CARES}/include
${CMAKE_CURRENT_BINARY_DIR}/${FLB_PATH_LIB_CARES}
)

add_subdirectory(${FLB_PATH_LIB_CARES})# EXCLUDE_FROM_ALL)

set(LIBCARES_LIBRARIES "c-ares")
10 changes: 0 additions & 10 deletions cmake/headers.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -32,26 +32,16 @@ include_directories(
${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_SNAPPY}
${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CMETRICS}/include
${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CTRACES}/include
${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CARES}/include
${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_RING_BUFFER}/lwrb/src/include

${CMAKE_CURRENT_BINARY_DIR}/${FLB_PATH_LIB_CARES}
${CMAKE_CURRENT_BINARY_DIR}/${FLB_PATH_LIB_JANSSON}/include
${CMAKE_CURRENT_BINARY_DIR}/lib/cmetrics
${CMAKE_CURRENT_BINARY_DIR}/include

${CMAKE_CURRENT_BINARY_DIR}/lib/monkey/include/
${CMAKE_CURRENT_BINARY_DIR}/lib/monkey/include/monkey/

${CMAKE_CURRENT_BINARY_DIR}/lib/nghttp2
${CMAKE_CURRENT_BINARY_DIR}/lib/nghttp2/lib/includes/
${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_NGHTTP2}/lib/includes/
)

if(FLB_IN_KAFKA OR FLB_OUT_KAFKA)
include_directories(${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_RDKAFKA}/src/)
endif()

if(FLB_UTF8_ENCODER)
include_directories(${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_TUTF8E}/include)
endif()
Expand Down
16 changes: 16 additions & 0 deletions cmake/kafka.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# kafka cmake
FLB_OPTION(RDKAFKA_BUILD_STATIC On)
FLB_OPTION(RDKAFKA_BUILD_EXAMPLES Off)
FLB_OPTION(RDKAFKA_BUILD_TESTS Off)
FLB_OPTION(ENABLE_LZ4_EXT Off)

# disable Curl
if (FLB_SYSTEM_MACOS)
FLB_OPTION(WITH_CURL Off)
endif()

include_directories(${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_RDKAFKA}/src/)

add_subdirectory(${FLB_PATH_LIB_RDKAFKA} EXCLUDE_FROM_ALL)

set(KAFKA_LIBRARIES "rdkafka")
12 changes: 12 additions & 0 deletions cmake/nghttp2.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# nghttp2 cmake
FLB_OPTION(ENABLE_LIB_ONLY ON)
FLB_OPTION(ENABLE_STATIC_LIB ON)
FLB_OPTION(ENABLE_SHARED_LIB OFF)
FLB_DEFINITION(NGHTTP2_STATICLIB)
include_directories(
${CMAKE_CURRENT_BINARY_DIR}/lib/nghttp2
${CMAKE_CURRENT_BINARY_DIR}/lib/nghttp2/lib/includes/
${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_NGHTTP2}/lib/includes/
)
add_subdirectory(${FLB_PATH_LIB_NGHTTP2} EXCLUDE_FROM_ALL)
set(NGHTTP2_LIBRARIES "nghttp2_static")
3 changes: 2 additions & 1 deletion plugins/in_kafka/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,6 @@ set(src
in_kafka.c
)

FLB_PLUGIN(in_kafka "${src}" "rdkafka")
FLB_PLUGIN(in_kafka "${src}" ${KAFKA_LIBRARIES})
target_include_directories(flb-plugin-in_kafka PUBLIC ${KAFKA_INCLUDEDIR}/librdkafka)
target_link_libraries(flb-plugin-in_kafka -lpthread)
3 changes: 2 additions & 1 deletion plugins/out_kafka/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,6 @@ set(src
kafka_topic.c
kafka.c)

FLB_PLUGIN(out_kafka "${src}" "rdkafka")
FLB_PLUGIN(out_kafka "${src}" ${KAFKA_LIBRARIES})
target_include_directories(flb-plugin-out_kafka PUBLIC ${KAFKA_INCLUDEDIR}/librdkafka)
target_link_libraries(flb-plugin-out_kafka -lpthread)
2 changes: 1 addition & 1 deletion plugins/processor_sql/parser/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ add_library(processor-sql-parser STATIC

add_flex_bison_dependency(lexer parser)

if (FLB_JEMALLOC)
if (FLB_JEMALLOC AND (NOT JEMALLOC_FOUND))
add_dependencies(processor-sql-parser libjemalloc)
include_directories("${CMAKE_BINARY_DIR}/include/")
endif ()
12 changes: 6 additions & 6 deletions src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ set(extra_libs
if(FLB_JEMALLOC)
set(extra_libs
${extra_libs}
"libjemalloc")
${JEMALLOC_LIBRARIES})
endif()

if(FLB_REGEX)
Expand Down Expand Up @@ -378,10 +378,10 @@ set(FLB_DEPS
${FLB_PLUGINS}
${FLB_PROXY_PLUGINS}
${extra_libs}
c-ares
${LIBCARES_LIBRARIES}
snappy-c
lwrb
nghttp2_static
${NGHTTP2_LIBRARIES}
)

if(OPENSSL_FOUND)
Expand Down Expand Up @@ -478,7 +478,7 @@ else()
endif(MSVC)

if(FLB_JEMALLOC)
target_link_libraries(fluent-bit-static libjemalloc)
target_link_libraries(fluent-bit-static ${JEMALLOC_LIBRARIES})
endif()

# Binary / Executable
Expand All @@ -501,12 +501,12 @@ if(FLB_BINARY)
endif()

if(FLB_JEMALLOC)
target_link_libraries(fluent-bit-bin libjemalloc)
target_link_libraries(fluent-bit-bin ${JEMALLOC_LIBRARIES})
endif()

if(FLB_BACKTRACE)
add_definitions(-DFLB_DUMP_STACKTRACE=1)
target_link_libraries(fluent-bit-bin libbacktrace)
target_link_libraries(fluent-bit-bin ${LIBBACKTRACE_LIBRARIES})
endif()

target_link_libraries(fluent-bit-bin fluent-bit-static ${CMAKE_THREAD_LIBS_INIT})
Expand Down
2 changes: 1 addition & 1 deletion src/aws/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -28,5 +28,5 @@ add_library(flb-aws STATIC ${src})
target_link_libraries(flb-aws flb-aws-compress)

if(FLB_JEMALLOC)
target_link_libraries(flb-aws libjemalloc)
target_link_libraries(flb-aws ${JEMALLOC_LIBRARIES})
endif()
Loading