fix: add missing dependencies for SASL support on all targets

.github/workflows/call-build-macos-packages.yaml

@@ -44,7 +44,7 @@ jobs:
               run: |
                   brew update
                   # We ignore errors here as some packages may already be installed
-                  brew install bison flex gnu-sed libyaml openssl pkgconfig libgit2 cyrus-sasl rocksdb || true
+                  brew install bison flex gnu-sed libyaml openssl pkgconfig libgit2 cyrus-sasl rocksdb libtool || true
                   # Remove dynamic linkage for libyaml
                   rm -fv $(brew --prefix libyaml)/lib/*.dylib
                   # Remove dynamic linkage for zstd added by curl install

.github/workflows/call-build-windows-packages.yaml

@@ -115,6 +115,11 @@ jobs:
           C:\vcpkg\vcpkg install --recurse libgit2 --triplet ${{ matrix.config.vcpkg_triplet }}
         shell: cmd
 
+      - name: Build libsasl with vcpkg
+        run: |
+          C:\vcpkg\vcpkg install --recurse cyrus-sasl --allow-unsupported --triplet ${{ matrix.config.vcpkg_triplet }}
+        shell: cmd
+
       - name: Upgrade any outdated vcpkg packages
         run: |
           C:\vcpkg\vcpkg upgrade --no-dry-run

build.sh

@@ -0,0 +1 @@
+source/packaging/build.sh

scripts/add-ignored-files.sh

@@ -15,7 +15,9 @@ while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
 done
 SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )
 SOURCE_DIR=${SOURCE_DIR:-$SCRIPT_DIR/../source}
-export FLUENT_BIT_VERSION=${FLUENT_BIT_VERSION:-v4.0.5}
+
+# Retrieve the version to use if not specified
+export FLUENT_BIT_VERSION=${FLUENT_BIT_VERSION:-$(cat "${SOURCE_DIR}"/oss_version.txt)}
 
 while IFS= read -r ignoredFile
 do

scripts/create-sync-pr.sh

@@ -7,8 +7,19 @@ set -euo pipefail
 # This script creates a GitHub PR for upstream sync commits
 # =============================================================================
 
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-REPO_ROOT="${SCRIPT_DIR}/.."
+# This does not work with a symlink to this script
+# SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+# See https://stackoverflow.com/a/246128/24637657
+SOURCE=${BASH_SOURCE[0]}
+while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )
+  SOURCE=$(readlink "$SOURCE")
+  # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+  [[ $SOURCE != /* ]] && SOURCE=$SCRIPT_DIR/$SOURCE
+done
+SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )
+
+REPO_ROOT=${REPO_ROOT:-${SCRIPT_DIR}/..}
 cd "$REPO_ROOT"
 
 # Colors for output

scripts/sync-upstream-patches.sh

@@ -11,8 +11,19 @@ set -euo pipefail
 # we use patch files to apply changes selectively.
 # =============================================================================
 
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-REPO_ROOT="${SCRIPT_DIR}/.."
+# This does not work with a symlink to this script
+# SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+# See https://stackoverflow.com/a/246128/24637657
+SOURCE=${BASH_SOURCE[0]}
+while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )
+  SOURCE=$(readlink "$SOURCE")
+  # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+  [[ $SOURCE != /* ]] && SOURCE=$SCRIPT_DIR/$SOURCE
+done
+SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )
+
+REPO_ROOT=${REPO_ROOT:-${SCRIPT_DIR}/..}
 cd "$REPO_ROOT" || exit 1
 
 # Configuration

source/CMakeLists.txt

@@ -193,6 +193,8 @@ option(FLB_CONFIG_YAML         "Enable YAML config format"    Yes)
 # List of plugins available and defaults for each option
 include(cmake/plugins_options.cmake)
 
+# Add any special Enterprise support
+include(cmake/enterprise.cmake)
 
 # Event loop backend (advanced users only)
 # ----------------------------------------
@@ -778,11 +780,11 @@ if(FLB_AWS AND FLB_KAFKA)
     FLB_DEFINITION(FLB_HAVE_AWS_MSK_IAM)  # Set preprocessor definition
     message(STATUS "AWS MSK IAM authentication: ENABLED")
   else()
-    message(STATUS "AWS MSK IAM authentication: DISABLED (requires Kafka with libsasl2)")
+    message(FATAL_ERROR "AWS MSK IAM authentication: DISABLED (requires Kafka with libsasl2)")
   endif()
 else()
   if(FLB_AWS AND NOT FLB_KAFKA)
-    message(STATUS "AWS MSK IAM authentication: DISABLED (requires FLB_KAFKA=ON)")
+    message(FATAL_ERROR "AWS MSK IAM authentication: DISABLED (requires FLB_KAFKA=ON)")
   endif()
 endif()
 
@@ -1589,3 +1591,5 @@ message(STATUS "Libraries install to: ${CMAKE_INSTALL_LIBDIR}")
 file(GLOB_RECURSE INSTALL_FILES "${CMAKE_INSTALL_PREFIX}/*")
 message(STATUS "Files to package: ${INSTALL_FILES}")
 
+# Verify all required options are set
+validate_required_options(FLB_HAVE_AWS_MSK_IAM FLB_HAVE_LIBYAML)

source/cmake/enterprise.cmake

@@ -5,3 +5,16 @@ FLB_DEFINITION(FLB_ENTERPRISE)
 if(CMAKE_INSTALL_PREFIX MATCHES "/opt/td-agent-bit")
   set(FLB_TD ON)
 endif()
+
+# Ensure we have specific options enabled (they may get disabled implicitly due to missing dependencies)
+function(validate_required_options)
+    set(REQUIRED_OPTIONS ${ARGV})
+
+    foreach(OPT ${REQUIRED_OPTIONS})
+        if(NOT ${OPT})
+            message(FATAL_ERROR "ERROR: ${OPT} is required but disabled.")
+        endif()
+    endforeach()
+
+    message(STATUS "All required options validated successfully")
+endfunction()

source/packaging/build.sh

@@ -1,13 +1,22 @@
 #!/bin/bash
 # Build a specific Linux target using the local source code via a container image
-set -eux
-
-# Never rely on PWD so we can invoke from anywhere
-SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+set -euo pipefail
+
+# This does not work with a symlink to this script
+# SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+# See https://stackoverflow.com/a/246128/24637657
+SOURCE=${BASH_SOURCE[0]}
+while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )
+  SOURCE=$(readlink "$SOURCE")
+  # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+  [[ $SOURCE != /* ]] && SOURCE=$SCRIPT_DIR/$SOURCE
+done
+SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )
 
 # Allow us to specify in the caller or pass variables
 FLB_DISTRO=${FLB_DISTRO:-}
-FLB_OUT_DIR=${FLB_OUT_DIR:-}
+FLB_OUT_DIR=${FLB_OUT_DIR:-agent}
 FLB_NIGHTLY_BUILD=${FLB_NIGHTLY_BUILD:-}
 DOCKER=${FLB_DOCKER_CLI:-docker}
 CACHE_ID=${CACHE_ID:-main}

source/packaging/distros/almalinux/Dockerfile

@@ -23,7 +23,7 @@ ARG CACHE_ID
 RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=almalinux-8-${CACHE_ID} yum -y update && \
     yum install -y rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
 
 # Split into separate layer to allow us to cache above
 RUN yum clean all
@@ -42,7 +42,7 @@ ARG CACHE_ID
 RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=almalinux-8-arm64-${CACHE_ID} yum -y update && \
     yum install -y rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
 
 # Split into separate layer to allow us to cache above
 RUN yum clean all
@@ -62,7 +62,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=almalinux-9-${CAC
     yum install -y epel-release && \
     yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
 
 # Split into separate layer to allow us to cache above
 RUN yum clean all
@@ -81,7 +81,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=almalinux-9-arm64
     yum install -y epel-release && \
     yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
 
 # Split into separate layer to allow us to cache above
 RUN yum clean all
@@ -101,7 +101,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=almalinux-10-${CA
     yum install -y epel-release && \
     yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
 
 # Split into separate layer to allow us to cache above
 RUN yum clean all
@@ -120,7 +120,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=almalinux-10-arm6
     yum install -y epel-release && \
     yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
 
 # Split into separate layer to allow us to cache above
 RUN yum clean all

source/packaging/distros/centos/6/Dockerfile

@@ -27,7 +27,7 @@ ARG CACHE_ID
 # hadolint ignore=DL3032, DL3033
 RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=centos-6-${CACHE_ID} yum -y update && \
     yum install -y rpm-build curl ca-certificates gcc gcc-c++ make wget tar \
-    openssl openss-libs openssl-devel cyrus-sasl-devel \
+    openssl openss-libs openssl-devel cyrus-sasl-lib cyrus-sasl-devel \
     libyaml-devel \
     perl-core libtemplate-perl zlib-devel \
     libicu-devel scl-utils-build devtoolset-9 rh-python35 && \

source/packaging/distros/centos/Dockerfile

@@ -62,9 +62,6 @@ RUN mkdir -p "${CMAKE_HOME}" && \
 
 ENV PATH="${CMAKE_HOME}/bin:${PATH}"
 
-ARG FLB_KAFKA=Off
-ENV FLB_KAFKA=${FLB_KAFKA}
-
 # centos/7.arm64v8 base image
 FROM arm64v8/centos:7 AS centos-7.arm64v8-base
 
@@ -120,9 +117,6 @@ ENV PATH="${CMAKE_HOME}/bin:${PATH}"
 ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3"
 ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS
 
-ARG FLB_KAFKA=Off
-ENV FLB_KAFKA=${FLB_KAFKA}
-
 # centos/8 base image
 FROM centos:8 AS centos-8-base
 
@@ -157,9 +151,6 @@ RUN mkdir -p "${CMAKE_HOME}" && \
 
 ENV PATH="${CMAKE_HOME}/bin:${PATH}"
 
-ARG FLB_KAFKA=On
-ENV FLB_KAFKA=${FLB_KAFKA}
-
 # centos/8.arm64v8 base image
 FROM arm64v8/centos:8 AS centos-8.arm64v8-base
 
@@ -200,9 +191,6 @@ ENV PATH="${CMAKE_HOME}/bin:${PATH}"
 ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3"
 ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS
 
-ARG FLB_KAFKA=On
-ENV FLB_KAFKA=${FLB_KAFKA}
-
 FROM quay.io/centos/centos:stream9 AS centos-9-base
 
 ENV CMAKE_HOME="/opt/cmake"
@@ -232,9 +220,6 @@ RUN mkdir -p "${CMAKE_HOME}" && \
 
 ENV PATH="${CMAKE_HOME}/bin:${PATH}"
 
-ARG FLB_KAFKA=On
-ENV FLB_KAFKA=${FLB_KAFKA}
-
 # hadolint ignore=DL3029
 FROM --platform=arm64 quay.io/centos/centos:stream9 AS centos-9.arm64v8-base
 
@@ -270,9 +255,6 @@ ENV PATH="${CMAKE_HOME}/bin:${PATH}"
 ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3"
 ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS
 
-ARG FLB_KAFKA=On
-ENV FLB_KAFKA=${FLB_KAFKA}
-
 FROM quay.io/centos/centos:stream10 AS centos-10-base
 
 ENV CMAKE_HOME="/opt/cmake"
@@ -301,9 +283,6 @@ RUN mkdir -p "${CMAKE_HOME}" && \
 
 ENV PATH="${CMAKE_HOME}/bin:${PATH}"
 
-ARG FLB_KAFKA=On
-ENV FLB_KAFKA=${FLB_KAFKA}
-
 # hadolint ignore=DL3029
 FROM --platform=arm64 quay.io/centos/centos:stream10 AS centos-10.arm64v8-base
 
@@ -339,9 +318,6 @@ ENV PATH="${CMAKE_HOME}/bin:${PATH}"
 ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3"
 ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS
 
-ARG FLB_KAFKA=On
-ENV FLB_KAFKA=${FLB_KAFKA}
-
 # Common build for all distributions now
 # hadolint ignore=DL3006
 FROM $BASE_BUILDER AS builder
@@ -362,7 +338,6 @@ RUN cmake -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \
     -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \
     -DFLB_NIGHTLY_BUILD="$FLB_NIGHTLY_BUILD" \
     -DFLB_JEMALLOC_OPTIONS="$FLB_JEMALLOC_OPTIONS" \
-    -DFLB_KAFKA="$FLB_KAFKA" \
     ../
 
 VOLUME [ "/output" ]

source/packaging/distros/rockylinux/Dockerfile

@@ -24,7 +24,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=rockylinux-8-${CA
     yum install -y epel-release && \
     yum install -y rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
 # Split into a separate layer to cache above
 RUN yum clean all
 
@@ -43,7 +43,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=rockylinux-8-arm6
     yum install -y epel-release && \
     yum install -y rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
 # Split into a separate layer to cache above
 RUN yum clean all
 
@@ -62,7 +62,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=rockylinux-9-${CA
     yum install -y epel-release && \
     yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
 # Split into a separate layer to cache above
 RUN yum clean all
 
@@ -80,7 +80,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=rockylinux-9-arm6
     yum install -y epel-release && \
     yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
 # Split into a separate layer to cache above
 RUN yum clean all
 
@@ -99,7 +99,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=rockylinux-10-${C
     yum install -y epel-release && \
     yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
 # Split into a separate layer to cache above
 RUN yum clean all
 
@@ -117,7 +117,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=rockylinux-10-arm
     yum install -y epel-release && \
     yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
 # Split into a separate layer to cache above
 RUN yum clean all