diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index a51e5b7d4e6..5b3c1baef75 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -13,13 +13,13 @@ on: - 1.8 pull_request: paths-ignore: - - '.github/**' - - 'dockerfiles/**' - - 'docker_compose/**' - - 'packaging/**' - - '.gitignore' - - 'appveyor.yml' - - 'examples/**' + - ".github/**" + - "dockerfiles/**" + - "docker_compose/**" + - "packaging/**" + - ".gitignore" + - "appveyor.yml" + - "examples/**" branches: - master - 4.1 @@ -122,7 +122,7 @@ jobs: wget https://packages.apache.org/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb sudo apt-get install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb sudo apt-get update - sudo apt-get install -y -V libarrow-glib-dev libparquet-glib-dev + sudo apt-get install -y -V libarrow-dev libparquet-dev - name: ${{ matrix.compiler.cc }} & ${{ matrix.compiler.cxx }} - ${{ matrix.flb_option }} run: | @@ -222,26 +222,26 @@ jobs: - name: Build and test with arm runners run: | - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 90 - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 90 - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 90 + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 90 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 90 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 90 - export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) )) - export FLB_OPTION="${{ matrix.config.flb_option }}" - export FLB_OMIT_OPTION="${{ matrix.config.omit_option }}" - export GLOBAL_OPTION="${{ matrix.config.global_option }}" - export FLB_UNIT_TEST_OPTION="${{ matrix.config.unit_test_option }}" - export FLB_OPT="${FLB_OPTION} ${GLOBAL_OPTION} ${FLB_UNIT_TEST_OPTION} ${FLB_OMIT_OPTION}" + export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) )) + export FLB_OPTION="${{ matrix.config.flb_option }}" + export FLB_OMIT_OPTION="${{ matrix.config.omit_option }}" + export GLOBAL_OPTION="${{ matrix.config.global_option }}" + export FLB_UNIT_TEST_OPTION="${{ matrix.config.unit_test_option }}" + export FLB_OPT="${FLB_OPTION} ${GLOBAL_OPTION} ${FLB_UNIT_TEST_OPTION} ${FLB_OMIT_OPTION}" - echo "CC = ${{ matrix.config.compiler_cc }}, CXX = ${{ matrix.config.compiler_cxx }}, FLB_OPT = $FLB_OPT" + echo "CC = ${{ matrix.config.compiler_cc }}, CXX = ${{ matrix.config.compiler_cxx }}, FLB_OPT = $FLB_OPT" - if [[ -n "${CMAKE_HOME}" ]]; then - export PATH="${CMAKE_HOME}/bin:${PATH}" - fi + if [[ -n "${CMAKE_HOME}" ]]; then + export PATH="${CMAKE_HOME}/bin:${PATH}" + fi - cmake ${FLB_OPT} ../ - make -j $nparallel - ctest -j $nparallel --build-run-dir . --output-on-failure + cmake ${FLB_OPT} ../ + make -j $nparallel + ctest -j $nparallel --build-run-dir . --output-on-failure working-directory: build env: CC: ${{ matrix.config.compiler_cc }} diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index c7d714d0e8f..25c2988cd84 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -21,6 +21,10 @@ FROM multiarch/qemu-user-static:x86_64-aarch64 AS qemu-arm64 FROM debian:trixie-slim AS builder-base +# EXTRA_CMAKE_FLAGS can be used to enable optional features like Parquet +ARG EXTRA_CMAKE_FLAGS +ENV EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS} + COPY --from=qemu-arm32 /usr/bin/qemu-arm-static /usr/bin/ COPY --from=qemu-arm64 /usr/bin/qemu-aarch64-static /usr/bin/ @@ -34,7 +38,7 @@ RUN mkdir -p /fluent-bit/bin /fluent-bit/etc /fluent-bit/log ENV DEBIAN_FRONTEND=noninteractive -# hadolint ignore=DL3008 +# hadolint ignore=DL3008,SC2015 RUN apt-get update && \ apt-get install -y --no-install-recommends \ build-essential \ @@ -54,6 +58,13 @@ RUN apt-get update && \ flex \ bison \ libyaml-dev \ + && if echo "${EXTRA_CMAKE_FLAGS}" | grep -q "FLB_PARQUET_ENCODER=On"; then \ + curl -fsSL https://packages.apache.org/artifactory/arrow/debian/apache-arrow-apt-source-latest-trixie.deb -o apache-arrow-apt-source.deb && \ + apt-get install -y -V ./apache-arrow-apt-source.deb && \ + apt-get update && \ + apt-get install -y -V libarrow-dev libparquet-dev && \ + rm -f apache-arrow-apt-source.deb; \ + fi \ && apt-get satisfy -y cmake "cmake (<< 4.0)" \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -122,12 +133,23 @@ FROM debian:trixie-slim AS deb-extractor COPY --from=qemu-arm32 /usr/bin/qemu-arm-static /usr/bin/ COPY --from=qemu-arm64 /usr/bin/qemu-aarch64-static /usr/bin/ +ARG EXTRA_CMAKE_FLAGS +ENV EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS} + # We download all debs locally then extract them into a directory we can use as the root for distroless. # We also include some extra handling for the status files that some tooling uses for scanning, etc. WORKDIR /tmp SHELL ["/bin/bash", "-o", "pipefail", "-c"] +# hadolint ignore=SC2015,DL3008 RUN apt-get update && \ - apt-get download \ + if echo "${EXTRA_CMAKE_FLAGS}" | grep -q "FLB_PARQUET_ENCODER=On"; then \ + apt-get install -y --no-install-recommends curl ca-certificates && \ + curl -fsSL https://packages.apache.org/artifactory/arrow/debian/apache-arrow-apt-source-latest-trixie.deb -o apache-arrow-apt-source.deb && \ + apt-get install -y -V ./apache-arrow-apt-source.deb && \ + apt-get update && \ + rm -f apache-arrow-apt-source.deb; \ + fi \ + && apt-get download \ libssl3t64 \ libcurl4t64 \ libnghttp2-14 \ @@ -167,6 +189,19 @@ RUN apt-get update && \ libyaml-0-2 \ libcap2 \ libldap2 \ + # MAINTAINER: Arrow SONAME format: libarrow{major*100+minor*10} (e.g., 23.0.0 -> libarrow2300) + # Only update when upgrading Arrow version or if packages become unavailable in apt source. + && if echo "${EXTRA_CMAKE_FLAGS}" | grep -q "FLB_PARQUET_ENCODER=On"; then \ + apt-get download \ + libarrow2300 \ + libparquet2300 \ + libsnappy1v5 \ + libabsl20240722 \ + libbz2-1.0 \ + libprotobuf32t64 \ + libthrift-0.19.0t64 \ + libxml2; \ + fi \ && \ mkdir -p /dpkg/var/lib/dpkg/status.d/ && \ for deb in *.deb; do \ @@ -231,9 +266,12 @@ LABEL description="Fluent Bit multi-architecture debug container image" \ COPY --from=qemu-arm32 /usr/bin/qemu-arm-static /usr/bin/ COPY --from=qemu-arm64 /usr/bin/qemu-aarch64-static /usr/bin/ + +ARG EXTRA_CMAKE_FLAGS +ENV EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS} ENV DEBIAN_FRONTEND=noninteractive -# hadolint ignore=DL3008 +# hadolint ignore=DL3008,SC2015 RUN apt-get update && \ apt-get install -y --no-install-recommends \ libssl3t64 \ @@ -263,6 +301,22 @@ RUN apt-get update && \ htop atop strace iotop sysstat ncdu logrotate hdparm pciutils psmisc tree pv \ make tar flex bison \ libssl-dev libsasl2-dev libsystemd-dev zlib1g-dev libpq-dev libyaml-dev postgresql-server-dev-all \ + # MAINTAINER: Keep Arrow packages in sync with deb-extractor stage. + && if echo "${EXTRA_CMAKE_FLAGS}" | grep -q "FLB_PARQUET_ENCODER=On"; then \ + curl -fsSL https://packages.apache.org/artifactory/arrow/debian/apache-arrow-apt-source-latest-trixie.deb -o apache-arrow-apt-source.deb && \ + apt-get install -y -V ./apache-arrow-apt-source.deb && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + libarrow2300 \ + libparquet2300 \ + libsnappy1v5 \ + libabsl20240722 \ + libbz2-1.0 \ + libprotobuf32t64 \ + libthrift-0.19.0t64 \ + libxml2 && \ + rm -f apache-arrow-apt-source.deb; \ + fi \ && apt-get satisfy -y cmake "cmake (<< 4.0)" \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/dockerfiles/Dockerfile.windows b/dockerfiles/Dockerfile.windows index e40800f2535..6d13a5ce521 100644 --- a/dockerfiles/Dockerfile.windows +++ b/dockerfiles/Dockerfile.windows @@ -16,6 +16,9 @@ ARG WINDOWS_VERSION=ltsc2025 # Builder Image - Windows Server Core FROM mcr.microsoft.com/windows/servercore:$WINDOWS_VERSION AS builder-base +# Unified build argument for extra CMake flags (e.g., -DFLB_PARQUET_ENCODER=On) +ARG EXTRA_CMAKE_FLAGS="" + SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] # Clean up any existing temp files and create fresh temp directory @@ -154,6 +157,16 @@ RUN vcpkg install --recurse libyaml --triplet x64-windows-static; ` Remove-Item -Path $env:TEMP\* -Recurse -Force -ErrorAction SilentlyContinue; ` Remove-Item -Path C:\dev\vcpkg\buildtrees -Recurse -Force -ErrorAction SilentlyContinue; +# Install Apache Arrow with Parquet support if enabled via EXTRA_CMAKE_FLAGS +RUN if ($env:EXTRA_CMAKE_FLAGS -like '*-DFLB_PARQUET_ENCODER=On*') { ` + Write-Host 'Installing Apache Arrow with Parquet support...'; ` + vcpkg install --recurse arrow[parquet] --triplet x64-windows-static; ` + Remove-Item -Path $env:TEMP\* -Recurse -Force -ErrorAction SilentlyContinue; ` + Remove-Item -Path C:\dev\vcpkg\buildtrees -Recurse -Force -ErrorAction SilentlyContinue; ` + } else { ` + Write-Host 'Skipping Apache Arrow installation (FLB_PARQUET_ENCODER not in EXTRA_CMAKE_FLAGS)'; ` + } + # Final vcpkg cleanup to remove all build artifacts and save space RUN Remove-Item -Path C:\dev\vcpkg\downloads -Recurse -Force -ErrorAction SilentlyContinue; ` Remove-Item -Path C:\dev\vcpkg\buildtrees -Recurse -Force -ErrorAction SilentlyContinue; ` @@ -167,11 +180,20 @@ WORKDIR /src/build COPY . /src/ ARG BUILD_PARALLEL=1 +ARG EXTRA_CMAKE_FLAGS + +# Optional feature detection: Add cmake paths based on EXTRA_CMAKE_FLAGS SHELL ["cmd", "/S", "/C"] RUN call "%MSVS_HOME%\VC\Auxiliary\Build\vcvars64.bat" && ` + set "OPTIONAL_CMAKE_ARGS=" && ` + (echo %EXTRA_CMAKE_FLAGS% | findstr /C:"-DFLB_PARQUET_ENCODER=On" >nul && ( ` + set "OPTIONAL_CMAKE_ARGS=%OPTIONAL_CMAKE_ARGS% -DArrow_DIR=C:\dev\vcpkg\packages\arrow_x64-windows-static\share\arrow -DParquet_DIR=C:\dev\vcpkg\packages\arrow_x64-windows-static\share\parquet" ` + ) || cd .) & ` cmake -G "NMake Makefiles" ` - -DOPENSSL_ROOT_DIR='C:\dev\vcpkg\packages\openssl_x64-windows-static' ` - -DFLB_LIBYAML_DIR='C:\dev\vcpkg\packages\libyaml_x64-windows-static' ` + -DOPENSSL_ROOT_DIR=C:\dev\vcpkg\packages\openssl_x64-windows-static ` + -DFLB_LIBYAML_DIR=C:\dev\vcpkg\packages\libyaml_x64-windows-static ` + %OPTIONAL_CMAKE_ARGS% ` + %EXTRA_CMAKE_FLAGS% ` -DFLB_SIMD=On ` -DCMAKE_BUILD_TYPE=Release ` -DFLB_SHARED_LIB=Off ` diff --git a/dockerfiles/README.md b/dockerfiles/README.md index 6d1e4221336..4afe9e6bcc1 100644 --- a/dockerfiles/README.md +++ b/dockerfiles/README.md @@ -21,24 +21,31 @@ With QEMU set up and buildkit support, you can build all targets in one simple c To set up for Ubuntu 20.04 development PC as an example: 1. Add QEMU: https://askubuntu.com/a/1369504 + ``` sudo add-apt-repository ppa:jacob/virtualisation sudo apt-get update && sudo apt-get install qemu qemu-user qemu-user-static ``` + 2. Install buildkit: https://docs.docker.com/buildx/working-with-buildx/#install + ``` wget https://github.com/docker/buildx/releases/download/v0.7.1/buildx-v0.7.1.linux-amd64 mv buildx-v0.7.1.linux-amd64 ~/.docker/cli-plugins/docker-buildx chmod a+x ~/.docker/cli-plugins/docker-buildx ``` + 3. Configure and use: https://stackoverflow.com/a/60667468 + ``` docker run --rm --privileged multiarch/qemu-user-static --reset -p yes docker buildx rm builder docker buildx create --name builder --use docker buildx inspect --bootstrap ``` + 4. Build Fluent Bit from the **root of the Git repo (not from this directory)**: + ``` docker buildx build --platform "linux/amd64,linux/arm64,linux/arm/v7,linux/s390x" --target=production -f dockerfiles/Dockerfile . ``` @@ -47,10 +54,25 @@ docker buildx build --platform "linux/amd64,linux/arm64,linux/arm/v7,linux/s390x 1. Checkout the branch you want, e.g. 1.8 for 1.8.X containers. 2. Build Fluent Bit from the **root of the Git repo (not from this directory)**: + ``` $ docker build -t fluent/fluent-bit --target=production -f dockerfiles/Dockerfile . ``` + +### Optional: Build with Parquet encoder support + +To enable Parquet encoding support (requires Apache Arrow/Parquet libraries), use the `EXTRA_CMAKE_FLAGS` build argument: + +``` +$ docker build -t fluent/fluent-bit --target=production \ + --build-arg EXTRA_CMAKE_FLAGS="-DFLB_PARQUET_ENCODER=On" \ + -f dockerfiles/Dockerfile . +``` + +Note: Enabling Parquet support will increase the image size by approximately 49MB due to Apache Arrow dependencies. + 3. Test the container. + ``` $ docker run --rm -it fluent/fluent-bit:latest ``` @@ -90,10 +112,11 @@ ______ _ _ ______ _ _ ___ _____ ## ghcr.io topology Containers are "staged" prior to release in the following ways to `ghcr.io`: -* `ghcr.io/fluent/fluent-bit` - official releases, identical to DockerHub -* `ghcr.io/fluent/fluent-bit/staging` - all architectures staging images used for testing prior to release -* `ghcr.io/fluent/fluent-bit/master` - x86_64/AMD64 only images built on each push to master, used for integration tests -* `ghcr.io/fluent/fluent-bit/pr-X` - x86_64/AMD64 only PR images where `X` is the PR number + +- `ghcr.io/fluent/fluent-bit` - official releases, identical to DockerHub +- `ghcr.io/fluent/fluent-bit/staging` - all architectures staging images used for testing prior to release +- `ghcr.io/fluent/fluent-bit/master` - x86_64/AMD64 only images built on each push to master, used for integration tests +- `ghcr.io/fluent/fluent-bit/pr-X` - x86_64/AMD64 only PR images where `X` is the PR number ## Windows @@ -109,20 +132,34 @@ More information is available at: In addition, metadata as defined in OCI image spec annotations, is leveraged in the generated image. This is the reason for the additional `--build-arg` parameters. ### Minimum set of build-args + ```powershell docker build --no-cache ` --build-arg WINDOWS_VERSION=ltsc2019 ` -t fluent/fluent-bit:master-windows -f ./dockerfiles/Dockerfile.windows . ``` +### Optional: Build with Parquet encoder support + +To enable Parquet encoding support on Windows, add the `EXTRA_CMAKE_FLAGS` build argument: + +```powershell +docker build --no-cache ` + --build-arg WINDOWS_VERSION=ltsc2019 ` + --build-arg EXTRA_CMAKE_FLAGS="-DFLB_PARQUET_ENCODER=On" ` + -t fluent/fluent-bit:master-windows -f ./dockerfiles/Dockerfile.windows . +``` + +Note: Parquet support uses vcpkg to install Apache Arrow with static linking. + ## Contact Feel free to join us on our Mailing List or IRC: - - Slack: http://slack.fluentd.org / channel #fluent-bit - - Mailing List: https://groups.google.com/forum/#!forum/fluent-bit - - IRC: irc.freenode.net #fluent-bit - - Twitter: http://twitter.com/fluentbit +- Slack: http://slack.fluentd.org / channel #fluent-bit +- Mailing List: https://groups.google.com/forum/#!forum/fluent-bit +- IRC: irc.freenode.net #fluent-bit +- Twitter: http://twitter.com/fluentbit ## License